I am facing an issue while running multithread. I need a solution similar to Goroutine in GoLang.
Here is the code that I need to run in multithread.
import aiohttp
import asyncio
async def get_response():
async with aiohttp.ClientSession() as session:
async with session.get('https://swapi.dev/api/starships/9/') as response:
print(await response.json())
def make_sync_req(subscriber):
while True:
asyncio.sleep(2)
print('running after sleep: ', subscriber)
loop = asyncio.get_event_loop()
loop.run_until_complete(get_response())
subscriber = ["subscriber1", "subscriber2", "subscribe3"]
for s in subscriber:
print(s)
make_sync_req(s)
print('wait here')
asyncio.sleep(500)
In the above code, I need to run multiple make_sync_req functions in parallel.
This issue is related to NATS where I need to subscribe to multiple subjects.
Please provide me with an approach to solve the following problem.
I tried the following code
import aiohttp
import asyncio
import threading
async def get_response():
async with aiohttp.ClientSession() as session:
async with session.get('https://swapi.dev/api/starships/9/') as response:
print(await response.json())
def make_sync_req(subscriber):
while True:
asyncio.sleep(2)
print('running after sleep: ', subscriber)
loop = asyncio.get_event_loop()
loop.run_until_complete(get_response())
subscriber = ["subscriber1", "subscriber2", "subscribe3"]
for s in subscriber:
print(s)
thread = threading.Thread(target=make_sync_req, args=(s,))
thread.daemon = True # Daemonize thread
thread.start()
print('wait here')
asyncio.sleep(500)
This gives the following error
python main.py
subscriber1
subscriber2
running after sleep: subscriber2
C:\Users\Venugopal_Hegde\go\src\test-python-async\main.py:14: DeprecationWarning: There is no current event loop
loop = asyncio.get_event_loop()
subscribe3
Exception in thread Thread-2 (make_sync_req):
Traceback (most recent call last):
running after sleep: subscribe3
File "C:\Python310\lib\threading.py", line 1009, in _bootstrap_inner
wait here
C:\Users\Venugopal_Hegde\go\src\test-python-async\main.py:12: RuntimeWarning: coroutine 'sleep' was never awaited
asyncio.sleep(2)
RuntimeWarning: Enable tracemalloc to get the object allocation traceback
running after sleep: subscriber1
Exception in thread Thread-3 (make_sync_req):
self.run()
Exception in thread Thread-1 (make_sync_req):
Traceback (most recent call last):
Traceback (most recent call last):
File "C:\Python310\lib\threading.py", line 1009, in _bootstrap_inner
File "C:\Python310\lib\threading.py", line 946, in run
File "C:\Python310\lib\threading.py", line 1009, in _bootstrap_inner
C:\Users\Venugopal_Hegde\go\src\test-python-async\main.py:27: RuntimeWarning: coroutine 'sleep' was never awaited
asyncio.sleep(500)
RuntimeWarning: Enable tracemalloc to get the object allocation traceback
self.run()
self._target(*self._args, **self._kwargs)
File "C:\Users\Venugopal_Hegde\go\src\test-python-async\main.py", line 14, in make_sync_req
File "C:\Python310\lib\threading.py", line 946, in run
Fatal Python error: _enter_buffered_busy: could not acquire lock for <_io.BufferedWriter name='<stderr>'> at interpreter shutdown, possibly due to daemon threads
Python runtime state: finalizing (tstate=0x00000276561ced40)
Current thread 0x00005568 (most recent call first):
<no Python frame>
Extension modules: multidict._multidict, yarl._quoting_c, aiohttp._helpers, aiohttp._http_writer, aiohttp._http_parser, aiohttp._websocket, frozenlist._frozenlist (total: 7)
If you want to run async function as a thread, you can use python-worker (link)
import aiohttp
import asyncio
import threading
async def get_response():
async with aiohttp.ClientSession() as session:
async with session.get('https://swapi.dev/api/starships/9/') as response:
print(await response.json())
#async_worker
async def make_sync_req(subscriber):
while True:
asyncio.sleep(2)
print('running after sleep: ', subscriber)
await get_response()
subscriber = ["subscriber1", "subscriber2", "subscribe3"]
for s in subscriber:
print(s)
asyncio.run(make_sync_req(s))
print('wait here')
asyncio.sleep(500)
Related
Found a good-looking realisation for making asynchronous HTTP requests in this article:
https://www.twilio.com/blog/asynchronous-http-requests-in-python-with-aiohttp
Here is the code I tried to use:
import aiohttp
import asyncio
import time
start_time = time.time()
async def get_pokemon(session, url):
async with session.get(url) as resp:
pokemon = await resp.json()
return pokemon['name']
async def main():
async with aiohttp.ClientSession() as session:
tasks = []
for number in range(1, 151):
url = f'https://pokeapi.co/api/v2/pokemon/{number}'
tasks.append(asyncio.ensure_future(get_pokemon(session, url)))
original_pokemon = await asyncio.gather(*tasks)
for pokemon in original_pokemon:
print(pokemon)
asyncio.run(main())
print("--- %s seconds ---" % (time.time() - start_time))
But it gives me this:
...
\--- 0.7400617599487305 seconds ---
Exception ignored in: \<function \_ProactorBasePipeTransport.__del__ at 0x000001DBBDB26200\>
Traceback (most recent call last):
File "C:\\Users\\wdgsy\\AppData\\Local\\Programs\\Python\\Python310\\lib\\asyncio\\proactor_events.py", line 116, in __del__
self.close()
File "C:\\Users\\wdgsy\\AppData\\Local\\Programs\\Python\\Python310\\lib\\asyncio\\proactor_events.py", line 108, in close
self.\_loop.call_soon(self.\_call_connection_lost, None)
File "C:\\Users\\wdgsy\\AppData\\Local\\Programs\\Python\\Python310\\lib\\asyncio\\base_events.py", line 750, in call_soon
self.\_check_closed()
File "C:\\Users\\wdgsy\\AppData\\Local\\Programs\\Python\\Python310\\lib\\asyncio\\base_events.py", line 515, in \_check_closed
raise RuntimeError('Event loop is closed')
RuntimeError: Event loop is closed
Exception ignored in: \<function \_ProactorBasePipeTransport.__del__ at 0x000001DBBDB26200\>
Traceback (most recent call last):
...
I also tried closing session manually and using asyncio.create_task for tasks list, both didn't change anything.
I am trying to run the below aiohttp client sample code straight out of aiohttp docs but it is throwing an exception.
import aiohttp
import asyncio
async def main():
async with aiohttp.ClientSession() as session:
async with session.get('http://python.org') as response:
print("Status:", response.status)
print("Content-type:", response.headers['content-type'])
html = await response.text()
print("Body:", html[:15], "...")
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
Exception
Traceback (most recent call last):
File "aiohttp-test.py", line 16, in <module>
loop.run_until_complete(main())
File "/Users/waseem/.pyenv/versions/3.8.4/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "aiohttp-test.py", line 6, in main
async with aiohttp.ClientSession() as session:
TypeError: __init__() missing 1 required positional argument: 'connector'
Versions of the required libs installed
Does anyone know what could be happening here?
I am trying setup a flag for my async loop. I tried using asyncio.Event but I get a RuntimeError
Here's my code:
import sys
import asyncio
import logging
root = logging.getLogger()
root.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setFormatter(
logging.Formatter('%(asctime)s - %(name)s - %(levelname)s %(message)s')
)
root.addHandler(ch)
global event
event = asyncio.Event()
class AsyncTest():
async def bg(self):
root.debug('Waiting for 10 secs')
await asyncio.sleep(10)
event.set()
root.debug('Setted Event')
async def waiter(self):
event.clear()
root.debug('Created task')
asyncio.create_task(self.bg())
await asyncio.sleep(0)
root.debug('waiting')
await event.wait()
root.debug('Finished Waiting.')
return
async def main():
obj = AsyncTest()
await obj.waiter()
asyncio.run(main())
The bg courotine does a periodic task. Therefore it has to be run in asyncio.create_task() .
The waiter coroutine is called at multiple points to wait until the event is set.
So while running this i get this error:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "C:\Users\kevin\AppData\Local\Programs\Python\Python37\lib\asyncio\runners.py", line 43, in run
return loop.run_until_complete(main)
File "C:\Users\kevin\AppData\Local\Programs\Python\Python37\lib\asyncio\base_events.py", line 587, in run_until_complete
return future.result()
File "C:\Python Programs\Python\KLibs\NeuralConnect\test\test.py", line 39, in main
await obj.waiter()
File "C:\Python Programs\Python\KLibs\NeuralConnect\test\test.py", line 32, in waiter
await event.wait()
File "C:\Users\kevin\AppData\Local\Programs\Python\Python37\lib\asyncio\locks.py", line 293, in wait
await fut
RuntimeError: Task <Task pending coro=<main() running at C:\Python Programs\Python\KLibs\NeuralConnect\test\test.py:39> cb=[_run_until_complete_cb() at C:\Users\kevin\AppData\Local\Programs\Python\Python37\lib\asyncio\base_events.py:157]> got Future <Future pending> attached to a different loop
can anyone point out what is wrong in my above implementation? Also how should I correct? Should abandon the asyncio.Event() part and then use a bool variable?
The problem using the bool flag variable is that I have to do:
cond = False
while not cond:
await asyncio.sleep(0.3)
I find this un-pythonic. Is there any better implementation of this, incase asyncio.Event isnt suitable?
Finally I recieved an answer to this problem from alex_normane who answered a question linked to this. It seems that the asyncio.Event object has to passed as parameter to the coroutine in create_task(). Plus the asyncio.Event object must be created within a coroutine as also stated by user4815162342.
The answer is here: https://stackoverflow.com/a/65353254/13574717
I wrote a telegram bot in python using the telepot library. The bot itself is working. Since I also have some other stuff I need to do periodically while running the bot (namely writing stuff in logs), I want the bot to run in its own thread.
The telegram bot's code is a file called telebot.py while the threading code is in one called plantbot.py
I first tried simply creating a new thread that calls my run_bot() function, which didn't work and gave me the following RuntimeError:
Exception in thread Thread-1:
Traceback (most recent call last):
File "/usr/lib/python3.5/threading.py", line 914, in _bootstrap_inner
self.run()
File "/usr/lib/python3.5/threading.py", line 862, in run
self._target(*self._args, **self._kwargs)
File "plantbot.py", line 12, in start_bot
telebot.run_bot()
File "/srv/plantbot/telebot.py", line 108, in run_bot
loop = asyncio.get_event_loop()
File "/usr/lib/python3.5/asyncio/events.py", line 671, in get_event_loop
return get_event_loop_policy().get_event_loop()
File "/usr/lib/python3.5/asyncio/events.py", line 583, in get_event_loop
% threading.current_thread().name)
RuntimeError: There is no current event loop in thread 'Thread-1'.
Here is the code I used for that:
plantbot.py
def start_bot():
telebot.run_bot()
bot = threading.Thread(target=start_bot, daemon=True)
bot.start()
print("Bot started...")
bot.join()
telebot.py
[lots of unrelated code]
bot = telepot.aio.Bot(TOKEN)
def run_bot():
loop = asyncio.get_event_loop()
loop.create_task(MessageLoop(bot, {'chat': on_chat_message, 'callback_query': on_callback_query}).run_forever())
print('Listening ...')
loop.run_forever()
I then tried around a little with the event loops until I ended up creating the event loop in my main file (plantbot.py) which looked like this:
plantbot.py
def start_bot(loop):
telebot.run_bot(loop)
loop = asyncio.new_event_loop()
bot = threading.Thread(target=start_bot, args=(loop,), daemon=True)
bot.start()
print("Bot started...")
bot.join()
telebot.py
[...]
bot = telepot.aio.Bot(TOKEN)
def run_bot(loop):
loop.create_task(MessageLoop(bot, {'chat': on_chat_message, 'callback_query': on_callback_query}).run_forever())
print('Listening ...')
loop.run_forever()
I also tried another variation:
plantbot.py
def start_bot(loop):
asyncio.set_event_loop(loop)
loop.create_task(telebot.task.run_forever())
loop.run_forever()
loop = asyncio.new_event_loop()
bot = threading.Thread(target=start_bot, args=(loop,), daemon=True)
bot.start()
print("Bot started...")
bot.join()
telebot.py
[...]
bot = telepot.aio.Bot(TOKEN)
task = MessageLoop(bot, {'chat': on_chat_message, 'callback_query': on_callback_query})
Unfortunately, none of these worked.
My question is thus, how do I properly run an event loop in its own designated thread? Specifically: How do I run a telegram bot in its own thread?
I did end up finding a solution, though it's not a satisfying one:
I am now running the bot in the main thread and put everything else into other threads. It does work that way around.
This question is different than Is there a way to use asyncio.Queue in multiple threads?
I have 2 asyncio event loops running in two different threads.
Thread1 produces data to Thread2 through a asyncio.Queue().
One of the threads throws an exception: got Future <Future pending> attached to a different loop
Now this is true because I have a single queue that I use in different loops. How can I share a queue between two loops, in two different threads?
Example code:
q = asyncio.Queue()
async def producer(q):
await asyncio.sleep(3)
q.put(1)
def prod_work(q):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(producer(q))
async def consumer(q):
await asyncio.sleep(3)
res = await q.get()
def cons_work(q):
loop2 = asyncio.new_event_loop()
asyncio.set_event_loop(loop2)
loop2.run_until_complete(consumer(q))
def worker(q):
# main thread - uses this threads loop
prod = threading.Thread(target=prod_work, args=(q,))
# separate thread - uses NEW loop
cons = threading.Thread(target=cons_work, args=(q,))
prod.start()
cons.start()
worker(q)
full stack trace:
Exception in thread Thread-2:
Traceback (most recent call last):
File "/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7/threading.py", line 917, in _bootstrap_inner
self.run()
File "/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7/threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "asyncio_examples.py", line 24, in cons_work
loop2.run_until_complete(consumer(q))
File "/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7/asyncio/base_events.py", line 568, in run_until_complete
return future.result()
File "asyncio_examples.py", line 18, in consumer
res = await q.get()
File "/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7/asyncio/queues.py", line 159, in get
await getter
RuntimeError: Task <Task pending coro=<consumer() running at asyncio_examples.py:18> cb=[_run_until_complete_cb() at /usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7/asyncio/base_events.py:150]> got Future <Future pending> attached to a different loop