This question is different than Is there a way to use asyncio.Queue in multiple threads?
I have 2 asyncio event loops running in two different threads.
Thread1 produces data to Thread2 through a asyncio.Queue().
One of the threads throws an exception: got Future <Future pending> attached to a different loop
Now this is true because I have a single queue that I use in different loops. How can I share a queue between two loops, in two different threads?
Example code:
q = asyncio.Queue()
async def producer(q):
await asyncio.sleep(3)
q.put(1)
def prod_work(q):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(producer(q))
async def consumer(q):
await asyncio.sleep(3)
res = await q.get()
def cons_work(q):
loop2 = asyncio.new_event_loop()
asyncio.set_event_loop(loop2)
loop2.run_until_complete(consumer(q))
def worker(q):
# main thread - uses this threads loop
prod = threading.Thread(target=prod_work, args=(q,))
# separate thread - uses NEW loop
cons = threading.Thread(target=cons_work, args=(q,))
prod.start()
cons.start()
worker(q)
full stack trace:
Exception in thread Thread-2:
Traceback (most recent call last):
File "/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7/threading.py", line 917, in _bootstrap_inner
self.run()
File "/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7/threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "asyncio_examples.py", line 24, in cons_work
loop2.run_until_complete(consumer(q))
File "/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7/asyncio/base_events.py", line 568, in run_until_complete
return future.result()
File "asyncio_examples.py", line 18, in consumer
res = await q.get()
File "/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7/asyncio/queues.py", line 159, in get
await getter
RuntimeError: Task <Task pending coro=<consumer() running at asyncio_examples.py:18> cb=[_run_until_complete_cb() at /usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7/asyncio/base_events.py:150]> got Future <Future pending> attached to a different loop
Related
I am facing an issue while running multithread. I need a solution similar to Goroutine in GoLang.
Here is the code that I need to run in multithread.
import aiohttp
import asyncio
async def get_response():
async with aiohttp.ClientSession() as session:
async with session.get('https://swapi.dev/api/starships/9/') as response:
print(await response.json())
def make_sync_req(subscriber):
while True:
asyncio.sleep(2)
print('running after sleep: ', subscriber)
loop = asyncio.get_event_loop()
loop.run_until_complete(get_response())
subscriber = ["subscriber1", "subscriber2", "subscribe3"]
for s in subscriber:
print(s)
make_sync_req(s)
print('wait here')
asyncio.sleep(500)
In the above code, I need to run multiple make_sync_req functions in parallel.
This issue is related to NATS where I need to subscribe to multiple subjects.
Please provide me with an approach to solve the following problem.
I tried the following code
import aiohttp
import asyncio
import threading
async def get_response():
async with aiohttp.ClientSession() as session:
async with session.get('https://swapi.dev/api/starships/9/') as response:
print(await response.json())
def make_sync_req(subscriber):
while True:
asyncio.sleep(2)
print('running after sleep: ', subscriber)
loop = asyncio.get_event_loop()
loop.run_until_complete(get_response())
subscriber = ["subscriber1", "subscriber2", "subscribe3"]
for s in subscriber:
print(s)
thread = threading.Thread(target=make_sync_req, args=(s,))
thread.daemon = True # Daemonize thread
thread.start()
print('wait here')
asyncio.sleep(500)
This gives the following error
python main.py
subscriber1
subscriber2
running after sleep: subscriber2
C:\Users\Venugopal_Hegde\go\src\test-python-async\main.py:14: DeprecationWarning: There is no current event loop
loop = asyncio.get_event_loop()
subscribe3
Exception in thread Thread-2 (make_sync_req):
Traceback (most recent call last):
running after sleep: subscribe3
File "C:\Python310\lib\threading.py", line 1009, in _bootstrap_inner
wait here
C:\Users\Venugopal_Hegde\go\src\test-python-async\main.py:12: RuntimeWarning: coroutine 'sleep' was never awaited
asyncio.sleep(2)
RuntimeWarning: Enable tracemalloc to get the object allocation traceback
running after sleep: subscriber1
Exception in thread Thread-3 (make_sync_req):
self.run()
Exception in thread Thread-1 (make_sync_req):
Traceback (most recent call last):
Traceback (most recent call last):
File "C:\Python310\lib\threading.py", line 1009, in _bootstrap_inner
File "C:\Python310\lib\threading.py", line 946, in run
File "C:\Python310\lib\threading.py", line 1009, in _bootstrap_inner
C:\Users\Venugopal_Hegde\go\src\test-python-async\main.py:27: RuntimeWarning: coroutine 'sleep' was never awaited
asyncio.sleep(500)
RuntimeWarning: Enable tracemalloc to get the object allocation traceback
self.run()
self._target(*self._args, **self._kwargs)
File "C:\Users\Venugopal_Hegde\go\src\test-python-async\main.py", line 14, in make_sync_req
File "C:\Python310\lib\threading.py", line 946, in run
Fatal Python error: _enter_buffered_busy: could not acquire lock for <_io.BufferedWriter name='<stderr>'> at interpreter shutdown, possibly due to daemon threads
Python runtime state: finalizing (tstate=0x00000276561ced40)
Current thread 0x00005568 (most recent call first):
<no Python frame>
Extension modules: multidict._multidict, yarl._quoting_c, aiohttp._helpers, aiohttp._http_writer, aiohttp._http_parser, aiohttp._websocket, frozenlist._frozenlist (total: 7)
If you want to run async function as a thread, you can use python-worker (link)
import aiohttp
import asyncio
import threading
async def get_response():
async with aiohttp.ClientSession() as session:
async with session.get('https://swapi.dev/api/starships/9/') as response:
print(await response.json())
#async_worker
async def make_sync_req(subscriber):
while True:
asyncio.sleep(2)
print('running after sleep: ', subscriber)
await get_response()
subscriber = ["subscriber1", "subscriber2", "subscribe3"]
for s in subscriber:
print(s)
asyncio.run(make_sync_req(s))
print('wait here')
asyncio.sleep(500)
I am trying setup a flag for my async loop. I tried using asyncio.Event but I get a RuntimeError
Here's my code:
import sys
import asyncio
import logging
root = logging.getLogger()
root.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setFormatter(
logging.Formatter('%(asctime)s - %(name)s - %(levelname)s %(message)s')
)
root.addHandler(ch)
global event
event = asyncio.Event()
class AsyncTest():
async def bg(self):
root.debug('Waiting for 10 secs')
await asyncio.sleep(10)
event.set()
root.debug('Setted Event')
async def waiter(self):
event.clear()
root.debug('Created task')
asyncio.create_task(self.bg())
await asyncio.sleep(0)
root.debug('waiting')
await event.wait()
root.debug('Finished Waiting.')
return
async def main():
obj = AsyncTest()
await obj.waiter()
asyncio.run(main())
The bg courotine does a periodic task. Therefore it has to be run in asyncio.create_task() .
The waiter coroutine is called at multiple points to wait until the event is set.
So while running this i get this error:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "C:\Users\kevin\AppData\Local\Programs\Python\Python37\lib\asyncio\runners.py", line 43, in run
return loop.run_until_complete(main)
File "C:\Users\kevin\AppData\Local\Programs\Python\Python37\lib\asyncio\base_events.py", line 587, in run_until_complete
return future.result()
File "C:\Python Programs\Python\KLibs\NeuralConnect\test\test.py", line 39, in main
await obj.waiter()
File "C:\Python Programs\Python\KLibs\NeuralConnect\test\test.py", line 32, in waiter
await event.wait()
File "C:\Users\kevin\AppData\Local\Programs\Python\Python37\lib\asyncio\locks.py", line 293, in wait
await fut
RuntimeError: Task <Task pending coro=<main() running at C:\Python Programs\Python\KLibs\NeuralConnect\test\test.py:39> cb=[_run_until_complete_cb() at C:\Users\kevin\AppData\Local\Programs\Python\Python37\lib\asyncio\base_events.py:157]> got Future <Future pending> attached to a different loop
can anyone point out what is wrong in my above implementation? Also how should I correct? Should abandon the asyncio.Event() part and then use a bool variable?
The problem using the bool flag variable is that I have to do:
cond = False
while not cond:
await asyncio.sleep(0.3)
I find this un-pythonic. Is there any better implementation of this, incase asyncio.Event isnt suitable?
Finally I recieved an answer to this problem from alex_normane who answered a question linked to this. It seems that the asyncio.Event object has to passed as parameter to the coroutine in create_task(). Plus the asyncio.Event object must be created within a coroutine as also stated by user4815162342.
The answer is here: https://stackoverflow.com/a/65353254/13574717
I wrote a telegram bot in python using the telepot library. The bot itself is working. Since I also have some other stuff I need to do periodically while running the bot (namely writing stuff in logs), I want the bot to run in its own thread.
The telegram bot's code is a file called telebot.py while the threading code is in one called plantbot.py
I first tried simply creating a new thread that calls my run_bot() function, which didn't work and gave me the following RuntimeError:
Exception in thread Thread-1:
Traceback (most recent call last):
File "/usr/lib/python3.5/threading.py", line 914, in _bootstrap_inner
self.run()
File "/usr/lib/python3.5/threading.py", line 862, in run
self._target(*self._args, **self._kwargs)
File "plantbot.py", line 12, in start_bot
telebot.run_bot()
File "/srv/plantbot/telebot.py", line 108, in run_bot
loop = asyncio.get_event_loop()
File "/usr/lib/python3.5/asyncio/events.py", line 671, in get_event_loop
return get_event_loop_policy().get_event_loop()
File "/usr/lib/python3.5/asyncio/events.py", line 583, in get_event_loop
% threading.current_thread().name)
RuntimeError: There is no current event loop in thread 'Thread-1'.
Here is the code I used for that:
plantbot.py
def start_bot():
telebot.run_bot()
bot = threading.Thread(target=start_bot, daemon=True)
bot.start()
print("Bot started...")
bot.join()
telebot.py
[lots of unrelated code]
bot = telepot.aio.Bot(TOKEN)
def run_bot():
loop = asyncio.get_event_loop()
loop.create_task(MessageLoop(bot, {'chat': on_chat_message, 'callback_query': on_callback_query}).run_forever())
print('Listening ...')
loop.run_forever()
I then tried around a little with the event loops until I ended up creating the event loop in my main file (plantbot.py) which looked like this:
plantbot.py
def start_bot(loop):
telebot.run_bot(loop)
loop = asyncio.new_event_loop()
bot = threading.Thread(target=start_bot, args=(loop,), daemon=True)
bot.start()
print("Bot started...")
bot.join()
telebot.py
[...]
bot = telepot.aio.Bot(TOKEN)
def run_bot(loop):
loop.create_task(MessageLoop(bot, {'chat': on_chat_message, 'callback_query': on_callback_query}).run_forever())
print('Listening ...')
loop.run_forever()
I also tried another variation:
plantbot.py
def start_bot(loop):
asyncio.set_event_loop(loop)
loop.create_task(telebot.task.run_forever())
loop.run_forever()
loop = asyncio.new_event_loop()
bot = threading.Thread(target=start_bot, args=(loop,), daemon=True)
bot.start()
print("Bot started...")
bot.join()
telebot.py
[...]
bot = telepot.aio.Bot(TOKEN)
task = MessageLoop(bot, {'chat': on_chat_message, 'callback_query': on_callback_query})
Unfortunately, none of these worked.
My question is thus, how do I properly run an event loop in its own designated thread? Specifically: How do I run a telegram bot in its own thread?
I did end up finding a solution, though it's not a satisfying one:
I am now running the bot in the main thread and put everything else into other threads. It does work that way around.
I want to run a function in background. so I use Threading in my code.
but return error ValueError: signal only works in main thread and don't know about two things:
what is the main thread
how to solve this problem :)
views.py
def callback(update):
print('I received', update)
def message_poll_start():
try:
client = TelegramClient('phone', api_id, api_hash,
update_workers=1, spawn_read_thread=False)
client.connect()
client.add_update_handler(callback)
client.idle()
except TypeNotFoundError:
pass
def message_poll_start_thread(request):
t = threading.Thread(target=message_poll_start, args=(), kwargs={})
t.setDaemon(True)
t.start()
return HttpResponse("message polling started")
urls.py
urlpatterns = [
path('message_poll_start', messagemanager_views.message_poll_start_thread, name="message_poll_start"),
]
trace
[12/Jan/2018 11:24:38] "GET /messages/message_poll_start HTTP/1.1" 200 23
Exception in thread Thread-3:
Traceback (most recent call last):
File "/usr/lib/python3.5/threading.py", line 914, in _bootstrap_inner
self.run()
File "/usr/lib/python3.5/threading.py", line 862, in run
self._target(*self._args, **self._kwargs)
File "/home/teletogram/telethogram/messagemanager/views.py", line 123, in message_poll_start
client0.idle()
File "/home/teletogram/.env/lib/python3.5/site-packages/telethon/telegram_bare_client.py", line 825, in idle
signal(sig, self._signal_handler)
File "/usr/lib/python3.5/signal.py", line 47, in signal
handler = _signal.signal(_enum_to_int(signalnum), _enum_to_int(handler))
ValueError: signal only works in main thread
1) A python script runs in the main thread by default. If you spawn a new thread using threading.Thread, that will create a new thread which runs separately from the main one. When I began learning about threading I spent a lot of time reading before it started to click. The official threading docs are decent for basic functionality, and I like this tutorial for a deeper dive.
2) The internals of Telethon rely on asyncio. In asyncio each thread needs its own asynchronous event loop, and thus spawned threads need an explicitly created event loop. Like threading, asyncio is a large topic, some of which is covered in the Telethon docs.
Something like this should work:
import asyncio
def message_poll_start():
try:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
client = TelegramClient('phone', api_id, api_hash, loop=loop)
client.connect()
client.add_update_handler(callback)
client.idle()
except TypeNotFoundError:
pass
I wrote a test program to try out using create_task() that needs to wait until the created task completes.
I tried using loop.run_until_complete() to wait for task completion, but it results in an error with a traceback.
/Users/jason/.virtualenvs/xxx/bin/python3.5 /Users/jason/asyncio/examples/hello_coroutine.py
Traceback (most recent call last):
Test
File "/Users/jason/asyncio/examples/hello_coroutine.py", line 42, in <module>
Hello World, is a task
loop.run_until_complete(test.greet_every_two_seconds())
File "/Users/jason/asyncio/asyncio/base_events.py", line 373, in run_until_complete
return future.result()
File "/Users/jason/asyncio/asyncio/futures.py", line 274, in result
raise self._exception
File "/Users/jason/asyncio/asyncio/tasks.py", line 240, in _step
result = coro.send(None)
File "/Users/jason/asyncio/examples/hello_coroutine.py", line 33, in greet_every_two_seconds
self.a()
File "/Users/jason/asyncio/examples/hello_coroutine.py", line 26, in a
t = loop.run_until_complete(self.greet_every_one_seconds(self.db_presell))
File "/Users/jason/asyncio/asyncio/base_events.py", line 361, in run_until_complete
self.run_forever()
File "/Users/jason/asyncio/asyncio/base_events.py", line 326, in run_forever
raise RuntimeError('Event loop is running.')
RuntimeError: Event loop is running.
The test code is as follows. The function a() must not be a coroutine,
How can I wait for the task until complete?
import asyncio
class Test(object):
def __init__(self):
print(self.__class__.__name__)
pass
#asyncio.coroutine
def greet_every_one_seconds(self, value):
print('Hello World, one second.')
fut = asyncio.sleep(1,result=value)
a = yield from fut
print(a)
def a(self):
loop = asyncio.get_event_loop()
task=loop.run_until_complete(self.greet_every_one_seconds(4))
#How can i wait for the task until complete?
#asyncio.coroutine
def greet_every_two_seconds(self):
while True:
self.a()
print('Hello World, two seconds.')
yield from asyncio.sleep(2)
if __name__ == '__main__':
test = Test()
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(test.greet_every_two_seconds())
finally:
loop.close()
How can i wait for the task until complete?
loop.run_until_complete(task) in an ordinary function. Or await task in a coroutine.
Here's a complete code example that demonstrates both cases:
#!/usr/bin/env python3
import asyncio
async def some_coroutine(loop):
task = loop.create_task(asyncio.sleep(1)) # just some task
await task # wait for it (inside a coroutine)
loop = asyncio.get_event_loop()
task = loop.create_task(asyncio.sleep(1)) # just some task
loop.run_until_complete(task) # wait for it (outside of a coroutine)
loop.run_until_complete(some_coroutine(loop))