My code is is similar to this example:
import discord
import tweepy
import asyncio
class Client(discord.Client):
async def on_ready(self):
print("ready")
global GUILD
GUILD = discord.utils.get(self.guilds, name = "Any Guild")
class TweepyStream(tweepy.Stream):
def on_connect(self):
print("connceted")
def on_status(self, status):
print(status.text)
channel = discord.utils.get(GUILD.channels, name = "twitter-posts")
asyncio.run(channel.send(status.text))
#From here the Discord message should be send
auth = tweepy.OAuthHandler(keys)
auth.set_access_token(tokens)
global api
api = tweepy.API(auth)
follow_list = []
follow = int(api.get_user(screen_name = "Any User").id)
print(follow)
follow_list.append(follow)
print(str(follow_list))
stream = TweepyStream(tokens and keys)
stream.filter(follow = follow_list, threaded=True) #track = track,
client = Client()
client.run(token)
I try to receive Twitter posts and send them into a Discord channel, but it doesn't work. How can I do this (maybe without asyncio)?
Or is there a way to work with a Python Twitter API, which works with async functions?
My error:
Stream encountered an exception
Traceback (most recent call last):
File "C:\Python39\lib\site-packages\tweepy\streaming.py", line 133, in _connect
self.on_data(line)
File "C:\Python39\lib\site-packages\tweepy\streaming.py", line 387, in on_data
return self.on_status(status)
File "c:\Users\morit\Desktop\FLL 2021 Bot\DiscordBot\DiscordBotV0.7\example.py", line 18, in on_status
asyncio.run(channel.send(status.text))
File "C:\Python39\lib\asyncio\runners.py", line 44, in run
return loop.run_until_complete(main)
File "C:\Python39\lib\asyncio\base_events.py", line 642, in run_until_complete
return future.result()
File "C:\Python39\lib\site-packages\discord\abc.py", line 1065, in send
data = await state.http.send_message(channel.id, content, tts=tts, embed=embed,
File "C:\Python39\lib\site-packages\discord\http.py", line 192, in request
async with self.__session.request(method, url, **kwargs) as r:
File "C:\Python39\lib\site-packages\aiohttp\client.py", line 1117, in __aenter__
self._resp = await self._coro
File "C:\Python39\lib\site-packages\aiohttp\client.py", line 448, in _request
with timer:
File "C:\Python39\lib\site-packages\aiohttp\helpers.py", line 635, in __enter__
raise RuntimeError(
RuntimeError: Timeout context manager should be used inside a task
asyncio.run creates a new event loop. You need to use the existing event loop that the discord.Client uses. You can retrieve this with Client.loop, asyncio.get_running_loop, or asyncio.get_event_loop.
You should probably also use asyncio.run_coroutine_threadsafe.
If you're using the current latest development version of Tweepy on the master branch, set to be released as v4.0, which it seems like you are, then you can also look into using AsyncStream
Related
i have a bot which parses some links given by user. When clients want to parse realy huge amount of links, bot parses them, creates csv file from those links, sends it to user(user can download and view this file) and then raise TimeoutError
Cause exception while getting updates.
Traceback (most recent call last):
File "/Users/alex26/miniforge3/envs/rq/lib/python3.8/site-packages/aiogram/dispatcher/dispatcher.py", line 381, in start_polling
updates = await self.bot.get_updates(
File "/Users/alex26/miniforge3/envs/rq/lib/python3.8/site-packages/aiogram/bot/bot.py", line 110, in get_updates
result = await self.request(api.Methods.GET_UPDATES, payload)
File "/Users/alex26/miniforge3/envs/rq/lib/python3.8/site-packages/aiogram/bot/base.py", line 231, in request
return await api.make_request(await self.get_session(), self.server, self.__token, method, data, files,
File "/Users/alex26/miniforge3/envs/rq/lib/python3.8/site-packages/aiogram/bot/api.py", line 139, in make_request
async with session.post(url, data=req, **kwargs) as response:
File "/Users/alex26/miniforge3/envs/rq/lib/python3.8/site-packages/aiohttp/client.py", line 1138, in __aenter__
self._resp = await self._coro
File "/Users/alex26/miniforge3/envs/rq/lib/python3.8/site-packages/aiohttp/client.py", line 559, in _request
await resp.start(conn)
File "/Users/alex26/miniforge3/envs/rq/lib/python3.8/site-packages/aiohttp/client_reqrep.py", line 913, in start
self._continue = None
File "/Users/alex26/miniforge3/envs/rq/lib/python3.8/site-packages/aiohttp/helpers.py", line 721, in __exit__
raise asyncio.TimeoutError from None
asyncio.exceptions.TimeoutError
Some example how my bot looks like(that's not real code, that's EXAMPLE):
bot = Bot(token=API_TOKEN)
dp = Dispatcher(bot)
def parse_1000_links():
#it takes about 10 mins and therefore that's the same like sleep(600)
sleep(600)
#dp.message_handler()
async def msg_handler(message: types.Message):
if msg == 'parse 1000 links':
res = parse_1000_links()
create_csv_from_res(res)
file_open_obj = open('data.csv', 'rb')
await bot.send_document(message.from_user.id, file_open_obj)
.....
if __name__ == '__main__':
executor.start_polling(dp, skip_updates=True)
User can even get from bot this final file, but after sending this file to user, bot raises this error. That's strange. If user get message, it means that everything is fine(That's my opinion)
How to fix my issue?
Thanks
You should avoid using blocking operations, cause they freeze ALL event loop.
If you can't use async version of your dependency, use executor:
https://docs.python.org/3/library/asyncio-eventloop.html#executing-code-in-thread-or-process-pools
Good day!
I am writing a telegram bot and using aiogram library.
Everything seems to be working fine when I run my code.
However, if I leave the bot running for a while, after some time it throws a timeout error. Can't understand what seems to be the issue - my PC falling asleep or something else?
I run an infinite loop like this
from aiogram import Bot, Dispatcher, types
from aiogram.contrib.fsm_storage.memory import MemoryStorage
import asyncio
import logging
from data import config
from utils.db_api.postgresql import Database
loop = asyncio.get_event_loop()
logging.basicConfig(format=u'%(filename)s [LINE:%(lineno)d] #%(levelname)-8s [%(asctime)s] %(message)s',
level=logging.INFO,
# level=logging.DEBUG,
)
bot = Bot(token=config.BOT_TOKEN, parse_mode=types.ParseMode.HTML)
storage = MemoryStorage()
dp = Dispatcher(bot, storage=storage)
db = Database()
if __name__ == '__main__':
executor.start_polling(dp, on_startup=on_startup)
the error is this
Traceback (most recent call last):
File "/Library/Python/3.8/site-packages/aiogram/dispatcher/dispatcher.py", line 381, in start_polling
updates = await self.bot.get_updates(
File "/Library/Python/3.8/site-packages/aiogram/bot/bot.py", line 110, in get_updates
result = await self.request(api.Methods.GET_UPDATES, payload)
File "/Library/Python/3.8/site-packages/aiogram/bot/base.py", line 231, in request
return await api.make_request(await self.get_session(), self.server, self.__token, method, data, files,
File "/Library/Python/3.8/site-packages/aiogram/bot/api.py", line 139, in make_request
async with session.post(url, data=req, **kwargs) as response:
File "/Library/Python/3.8/site-packages/aiohttp/client.py", line 1138, in __aenter__
self._resp = await self._coro
File "/Library/Python/3.8/site-packages/aiohttp/client.py", line 559, in _request
await resp.start(conn)
File "/Library/Python/3.8/site-packages/aiohttp/client_reqrep.py", line 913, in start
self._continue = None
File "/Library/Python/3.8/site-packages/aiohttp/helpers.py", line 721, in __exit__
raise asyncio.TimeoutError from None
asyncio.exceptions.TimeoutError
My code is simple because I am new to coding my discord bot, I want it so the bot sends a random meme from a subreddit when the word !meme is sent in discord, here is the code:
import discord
import os
import praw
import random
client = discord.Client()
reddit = praw.Reddit(client_id='the client id',
client_secret='the client secret',
user_agent='Memri TV Bot by /u/Hezbolloli')
#client.event
async def on_ready():
print('We have logged in as {0.user}'.format(client))
#client.event
async def on_message(message):
if message.author == client.user:
return
if message.content.startswith('$hello'):
subreddit = reddit.subreddit("memritvmemes")
all_subs = []
top = subreddit.hot(limit=50)
for submission in top:
all_subs.append(submission)
random_sub = random.choice(all_subs)
name = random_sub.title
url = random_sub.url
em = discord.Embed(title=name)
em.set_image(url=url)
await ctx.send(embed=em)
client.run('Token')
My error is here and I am not sure where I should start looking at this because this is by far the longest error I have ever got in my coding career:
It appears that you are using PRAW in an asynchronous environment.
It is strongly recommended to use Async PRAW: https://asyncpraw.readthedocs.io.
See https://praw.readthedocs.io/en/latest/getting_started/multiple_instances.html#discord-bots-and-asynchronous-environments for more info.
Ignoring exception in on_message
Traceback (most recent call last):
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/discord/client.py", line 343, in _run_event
await coro(*args, **kwargs)
File "main.py", line 28, in on_message
for submission in top:
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/praw/models/listing/generator.py", line 63, in __next__
self._next_batch()
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/praw/models/listing/generator.py", line 73, in _next_batch
self._listing = self._reddit.get(self.url, params=self.params)
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/praw/reddit.py", line 566, in get
return self._objectify_request(method="GET", params=params, path=path)
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/praw/reddit.py", line 666, in _objectify_request
self.request(
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/praw/reddit.py", line 848, in request
return self._core.request(
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/prawcore/sessions.py", line 324, in request
return self._request_with_retries(
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/prawcore/sessions.py", line 222, in _request_with_retries
response, saved_exception = self._make_request(
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/prawcore/sessions.py", line 179, in _make_request
response = self._rate_limiter.call(
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/prawcore/rate_limit.py", line 33, in call
kwargs["headers"] = set_header_callback()
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/prawcore/sessions.py", line 277, in _set_header_callback
self._authorizer.refresh()
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/prawcore/auth.py", line 346, in refresh
self._request_token(grant_type="client_credentials")
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/prawcore/auth.py", line 155, in _request_token
response = self._authenticator._post(url, **data)
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/prawcore/auth.py", line 38, in _post
raise ResponseException(response)
prawcore.exceptions.ResponseException: received 401 HTTP response
It seems the problem lies in your request to Reddit. As you are working in a asynchronous environment, you need to use the asyncpraw module instead (docs: Asyncpraw for reddit docs). The code is almost the same:
import discord
import os
import asyncpraw # install it using "pip install asyncpraw"
import random
client = discord.Client()
reddit = asyncpraw.Reddit(client_id='the client id',
client_secret='the client secret',
user_agent='Memri TV Bot by /u/Hezbolloli')
Also, you did not define ctx. Try : message.channel.send(...), or setup a bot.
If you are coding a bot, I would also strongly recommend that you use the discord bot commands as a way of adding functions, instead of reading the messages content (you could also use slash commands).
Here is a link to the docs : https://discordpy.readthedocs.io/en/stable/#getting-started
There is quite a lot of info about that online. If you run into trouble I'd be glad to help.
I have a FastAPI endpoint where it need to download some files from HDFS to the local server.
I'm trying to use asyncio to run the function that will download the files in a separate process.
I'm using FastAPI Depends to create a HDFS client and inject the object in the endpoint execution.
from fastapi import Depends, FastAPI, Request, Response, status
from hdfs import InsecureClient
import asyncio
from concurrent.futures.process import ProcessPoolExecutor
app = FastAPI()
HDFS_URLS = ['http://hdfs-srv.local:50070']
async def run_in_process(fn, *args):
loop = asyncio.get_event_loop()
return await loop.run_in_executor(app.state.executor, fn, *args) # wait and return result
def connectHDFS():
client = InsecureClient(url)
yield client
def fr(id, img, client):
# my code here
client.download(id_identifica_foto_dir_hdfs, id_identifica_foto_dir_local, True, n_threads=2)
# my code here
return jsonReturn
#app.post("/")
async def main(request: Request, hdfsclient: InsecureClient = Depends(connectHDFS)):
# Decode the received message
data = await request.json()
message = base64.b64decode(data['data']).decode('utf-8').replace("'", '"')
message = json.loads(message)
res = await run_in_process(fr, message['id'], message['img'], hdfsclient)
return {
"message": res
}
#app.on_event("startup")
async def on_startup():
app.state.executor = ProcessPoolExecutor()
#app.on_event("shutdown")
async def on_shutdown():
app.state.executor.shutdown()
But I'm not able to pass ahead the hdfsclient object:
res = await run_in_process(fr, message['id'], message['img'], hdfsclient)
I'm getting the following error:
Traceback (most recent call last):
File "/home/kleyson/.virtualenvs/reconhecimentofacial/lib/python3.7/site-packages/uvicorn/protocols/http/h11_impl.py", line 396, in run_asgi
result = await app(self.scope, self.receive, self.send)
File "/home/kleyson/.virtualenvs/reconhecimentofacial/lib/python3.7/site-packages/uvicorn/middleware/proxy_headers.py", line 45, in __call__
return await self.app(scope, receive, send)
File "/home/kleyson/.virtualenvs/reconhecimentofacial/lib/python3.7/site-packages/fastapi/applications.py", line 199, in __call__
await super().__call__(scope, receive, send)
File "/home/kleyson/.virtualenvs/reconhecimentofacial/lib/python3.7/site-packages/starlette/applications.py", line 111, in __call__
await self.middleware_stack(scope, receive, send)
File "/home/kleyson/.virtualenvs/reconhecimentofacial/lib/python3.7/site-packages/starlette/middleware/errors.py", line 181, in __call__
raise exc from None
File "/home/kleyson/.virtualenvs/reconhecimentofacial/lib/python3.7/site-packages/starlette/middleware/errors.py", line 159, in __call__
await self.app(scope, receive, _send)
File "/home/kleyson/.virtualenvs/reconhecimentofacial/lib/python3.7/site-packages/starlette/exceptions.py", line 82, in __call__
raise exc from None
File "/home/kleyson/.virtualenvs/reconhecimentofacial/lib/python3.7/site-packages/starlette/exceptions.py", line 71, in __call__
await self.app(scope, receive, sender)
File "/home/kleyson/.virtualenvs/reconhecimentofacial/lib/python3.7/site-packages/starlette/routing.py", line 566, in __call__
await route.handle(scope, receive, send)
File "/home/kleyson/.virtualenvs/reconhecimentofacial/lib/python3.7/site-packages/starlette/routing.py", line 227, in handle
await self.app(scope, receive, send)
File "/home/kleyson/.virtualenvs/reconhecimentofacial/lib/python3.7/site-packages/starlette/routing.py", line 41, in app
response = await func(request)
File "/home/kleyson/.virtualenvs/reconhecimentofacial/lib/python3.7/site-packages/fastapi/routing.py", line 202, in app
dependant=dependant, values=values, is_coroutine=is_coroutine
File "/home/kleyson/.virtualenvs/reconhecimentofacial/lib/python3.7/site-packages/fastapi/routing.py", line 148, in run_endpoint_function
return await dependant.call(**values)
File "./asgi.py", line 86, in main
res = await run_in_process(fr, message['id'], message['img'], hdfsclient)
File "./asgi.py", line 22, in run_in_process
return await loop.run_in_executor(app.state.executor, fn, *args) # wait and return result
File "/usr/lib/python3.7/multiprocessing/queues.py", line 236, in _feed
obj = _ForkingPickler.dumps(obj)
File "/usr/lib/python3.7/multiprocessing/reduction.py", line 51, in dumps
cls(buf, protocol).dump(obj)
TypeError: can't pickle _thread.lock objects
How can have the hdfsclient available inside the def fr() function without the need to create a new connection on every new request ? I mean, how to create the hdfsclient on the application startup and be able to use it inside the function ?
The entire point of asyncio is to do what you are trying to achieve in the same process.
The typical example is a web crawler, where you open multiple requests, within the same thread/process, and then wait for them to finish. This way, you will get data from multiple urls without having to wait each single request before starting the next one.
The same applies in your case: call your async function that downloads the file, do your stuff and then wait for the file download to complete (if it hasn't completed yet). Sharing data between processes is not trivial, and your function is not working because of that reason.
I suggest you to first understand what async is and how it works, before jumping into doing something that you don't understand.
Some tutorials on asyncio
https://www.datacamp.com/community/tutorials/asyncio-introduction
https://realpython.com/lessons/what-asyncio/
https://docs.python.org/3/library/asyncio.html
I am using a third party API Wrapper that supports async. I am having a difficult time figuring out how to use it as I am new to asyncio, and I'm not sure if the errors I'm getting are my usage (most likely) or the wrapper itself.
The API can be found here: https://clashroyale.readthedocs.io/en/latest/api.html#royaleapi
The goal of my code is to make three API requests at once (the .get_clan(*lClanGroup)), asynchronously instead of synchronously.
My Code:
import clashroyale
import asyncio
import aiohttp
# Define Tokens
unofficialAPIToken = "secretToken"
# Get Client Objects
session1 = aiohttp.ClientSession()
unofficialClient = clashroyale.royaleapi.Client(unofficialAPIToken, session=session1, is_async=True)
lClanGroups = ['PPCLCJG9', '2LRU2J', 'PGLQ0VQ', 'YU2RQG9', '2LVRQ29'],['PYP8UPJV', 'P9L0CYY0', 'Y2RGQPJ', '8P2GYJ8', '9VQJPL2L'],['RYPRGCJ', '809R8PG8', 'PJY9PP98', '2GCQLC', '2GL2QPPL']
async def x(lClanGroup):
print(*lClanGroup)
a = await unofficialClient.get_clan(*lClanGroup) # Iterates five tags for the API to request at once
return a
async def y(lClanGroups):
result = await asyncio.gather(x(lClanGroups[0]),x(lClanGroups[1]),x(lClanGroups[2]))
return result
async def close_sessions():
await session1.close()
asyncio.run(y(lClanGroups))
asyncio.run(close_sessions())
The error I'm getting is long and difficult to make sense of:
C:\Users\Adam\PycharmProjects\ClashRoyaleMeta\venv2\Scripts\python.exe C:/Users/Adam/PycharmProjects/ClashRoyaleMeta/testme2.py
PPCLCJG9 2LRU2J PGLQ0VQ YU2RQG9 2LVRQ29
PYP8UPJV P9L0CYY0 Y2RGQPJ 8P2GYJ8 9VQJPL2L
RYPRGCJ 809R8PG8 PJY9PP98 2GCQLC 2GL2QPPL
C:/Users/Adam/PycharmProjects/ClashRoyaleMeta/testme2.py:16: DeprecationWarning: The object should be created from async function
session1 = aiohttp.ClientSession()
Traceback (most recent call last):
File "C:/Users/Adam/PycharmProjects/ClashRoyaleMeta/testme2.py", line 33, in <module>
asyncio.run(y(lClanGroups))
File "C:\Program Files\Python38\lib\asyncio\runners.py", line 43, in run
return loop.run_until_complete(main)
File "C:\Program Files\Python38\lib\asyncio\base_events.py", line 612, in run_until_complete
return future.result()
File "C:/Users/Adam/PycharmProjects/ClashRoyaleMeta/testme2.py", line 27, in y
result = await asyncio.gather(x(lClanGroups[0]),x(lClanGroups[1]),x(lClanGroups[2]))
File "C:/Users/Adam/PycharmProjects/ClashRoyaleMeta/testme2.py", line 23, in x
a = await unofficialClient.get_clan(*lClanGroup)
File "C:\Users\Adam\PycharmProjects\ClashRoyaleMeta\venv2\lib\site-packages\clashroyale\royaleapi\client.py", line 203, in _aget_model
raise e
File "C:\Users\Adam\PycharmProjects\ClashRoyaleMeta\venv2\lib\site-packages\clashroyale\royaleapi\client.py", line 196, in _aget_model
data, cached, ts, resp = await self._request(url, **params)
File "C:\Users\Adam\PycharmProjects\ClashRoyaleMeta\venv2\lib\site-packages\clashroyale\royaleapi\client.py", line 150, in _arequest
async with self.session.get(url, timeout=timeout, headers=self.headers, params=params) as resp:
File "C:\Users\Adam\PycharmProjects\ClashRoyaleMeta\venv2\lib\site-packages\aiohttp\client.py", line 1012, in __aenter__
self._resp = await self._coro
File "C:\Users\Adam\PycharmProjects\ClashRoyaleMeta\venv2\lib\site-packages\aiohttp\client.py", line 426, in _request
with timer:
File "C:\Users\Adam\PycharmProjects\ClashRoyaleMeta\venv2\lib\site-packages\aiohttp\helpers.py", line 579, in __enter__
raise RuntimeError('Timeout context manager should be used '
RuntimeError: Timeout context manager should be used inside a task
Unclosed client session
client_session: <aiohttp.client.ClientSession object at 0x0000016C5AD50EB0>
Process finished with exit code 1
Thanks for your help!
Each asyncio.run() creates new event loop. Moreover, if you create objects like aiohttp.ClientSession() globally, they may be bind to default event loop.
Using multiple event loops in your program is dangerous and may lead to non-obvious problems. To avoid the situation:
Run asyncio.run() only once
Create all asyncio-related stuff inside main coroutine
I didn't work with royaleapi, but it seems API developers do things differently than you in their tests. Here's how they innit/close stuff and here's how they get clan.
Let's give it a try?
import clashroyale
import asyncio
TOKEN = "secretToken"
clan_groups = [
['PPCLCJG9', '2LRU2J', 'PGLQ0VQ', 'YU2RQG9', '2LVRQ29'],
['PYP8UPJV', 'P9L0CYY0', 'Y2RGQPJ', '8P2GYJ8', '9VQJPL2L'],
['RYPRGCJ', '809R8PG8', 'PJY9PP98', '2GCQLC', '2GL2QPPL']
]
async def get_clans(cr, clan_groups):
return await asyncio.gather(*[
cr.get_clan(tag)
for group in clan_groups
for tag in group
])
async def main():
cr = clashroyale.RoyaleAPI(
TOKEN,
is_async=True,
timeout=30
)
try:
results = await get_clans(cr, clan_groups)
print(results)
finally:
await cr.close()
await asyncio.sleep(2)
asyncio.run(main())
Didn't test it.