I would like to have a similar code to the following:
async def func(user_response):
#does_something
while condition:
#waits for a response from the user
#await func(response from user)
However, I want it to wait for another response from the user while the function is executing.
I have tried:
async def func(user_response):
#does_something
while condition:
#waits for a response from the user
#asyncio.create_task(response from user)
However, the problem i have found is that if the user responds twice, both functions will be carried out at the same time (when i was them to essentially be queued).
Can async tasks be queued
Yes, and you can use asyncio.Queue for that purpose:
async def func(user_response):
#does_something
async def worker(queue):
while True:
user_response = await queue.get()
await func(user_response)
async def main():
queue = asyncio.Queue()
# spawn the worker in the "background"
asyncio.create_task(worker(queue))
while condition:
#waits for a response from the user
# enqueue the response to be handled by the worker
await queue.put(user_response)
Related
async def main():
uuids = await get_uuids_from_text_file()
tasks = []
# create a task for each uuid
# and add it to the list of tasks
for uuid in uuids:
task = asyncio.create_task(make_hypixel_request(uuid))
tasks.append(task)
# wait for all the tasks to finish
responses = await asyncio.gather(*tasks)
# run the functions to process the data
for response in responses:
print(response['success'])
data2 = await anti_sniper_request(response)
await store_data_in_json_file(response, data2)
await compare_stats(response)
# loop the main function
async def main_loop():
for _ in itertools.repeat([]):
await main()
# run the loop
loop = asyncio.get_event_loop()
loop.run_until_complete(main_loop())
loop.close()
basically this is my code the functions have very clear and explanatory name
the make_hypixel_requests part i have no issues there, the requests are executed immediately and in parallel,
the problem is after that when "for response in responses" it goes hella slow? how do i get the responses instantly and loop through them very fast? i will try to attach a gif.
basically this is the issue:
the reason is because after waiting for all the responses to return, u process them in a loop instead of asynchronously, since none of the requests seem to depend on each other, waiting for them all to finish before processing them doesn't make sense, the best way to handle this is to couple the request and the processing e.g.
async def request_and_process(uuid):
response = await make_hypixel_request(uuid)
print(response['success'])
compare_stats_task = asyncio.create_task(compare_stats(response))
data2 = await anti_sniper_request(response)
await asyncio.gather(store_data_in_json_file(response, data2), compare_stats_task)
async def main():
while True:
uuids = await get_uuids_from_text_file()
await asyncio.gather(*map(request_and_process, uuids))
asyncio.run(main())
You can use asyncio.wait and return when at least a task is completed, then continue awaiting for the pending tasks. asyncio.wait return a tuple with two sets, the first with the completed tasks, the second with the still pending tasks. You can call to the result method of the done tasks and get its return value.
async def main():
uuids = await get_uuids_from_text_file()
tasks = []
# create a task for each uuid
# and add it to the list of tasks
for uuid in uuids:
task = asyncio.create_task(make_hypixel_request(uuid))
tasks.append(task)
# wait for all the tasks to finish
while tasks:
done_tasks, tasks = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
for done in done_tasks:
response = done.result()
print(response['success'])
data2 = await anti_sniper_request(response)
await store_data_in_json_file(response, data2)
await compare_stats(response)
I need to be able to keep adding coroutines to the asyncio loop at runtime. I tried using create_task() thinking that this would do what I want, but it still needs to be awaited.
This is the code I had, not sure if there is a simple edit to make it work?
async def get_value_from_api():
global ASYNC_CLIENT
return ASYNC_CLIENT.get(api_address)
async def print_subs():
count = await get_value_from_api()
print(count)
async def save_subs_loop():
while True:
asyncio.create_task(print_subs())
time.sleep(0.1)
async def start():
global ASYNC_CLIENT
async with httpx.AsyncClient() as ASYNC_CLIENT:
await save_subs_loop()
asyncio.run(start())
I once created similar pattern when I was mixing trio and kivy, which was demonstration of running multiple coroutines asynchronously.
It use a trio.MemoryChannel which is roughly equivalent to asyncio.Queue, I'll just refer it as queue here.
Main idea is:
Wrap each task with class, which has run function.
Make class object's own async method to put object itself into queue when execution is done.
Create a global task-spawning loop to wait for the object in queue and schedule execution/create task for the object.
import asyncio
import traceback
import httpx
async def task_1(client: httpx.AsyncClient):
resp = await client.get("http://127.0.0.1:5000/")
print(resp.read())
await asyncio.sleep(0.1) # without this would be IP ban
async def task_2(client: httpx.AsyncClient):
resp = await client.get("http://127.0.0.1:5000/meow/")
print(resp.read())
await asyncio.sleep(0.5)
class CoroutineWrapper:
def __init__(self, queue: asyncio.Queue, coro_func, *param):
self.func = coro_func
self.param = param
self.queue = queue
async def run(self):
try:
await self.func(*self.param)
except Exception:
traceback.print_exc()
return
# put itself back into queue
await self.queue.put(self)
class KeepRunning:
def __init__(self):
# queue for gathering CoroutineWrapper
self.queue = asyncio.Queue()
def add_task(self, coro, *param):
wrapped = CoroutineWrapper(self.queue, coro, *param)
# add tasks to be executed in queue
self.queue.put_nowait(wrapped)
async def task_processor(self):
task: CoroutineWrapper
while task := await self.queue.get():
# wait for new CoroutineWrapper Object then schedule it's async method execution
asyncio.create_task(task.run())
async def main():
keep_running = KeepRunning()
async with httpx.AsyncClient() as client:
keep_running.add_task(task_1, client)
keep_running.add_task(task_2, client)
await keep_running.task_processor()
asyncio.run(main())
Server
import time
from flask import Flask
app = Flask(__name__)
#app.route("/")
def hello():
return str(time.time())
#app.route("/meow/")
def meow():
return "meow"
app.run()
Output:
b'meow'
b'1639920445.965701'
b'1639920446.0767004'
b'1639920446.1887035'
b'1639920446.2986999'
b'1639920446.4067013'
b'meow'
b'1639920446.516704'
b'1639920446.6267014'
...
You can see tasks running repeatedly on their own pace.
Old answer
Seems like you only want to cycle fixed amount of tasks.
In that case just iterate list of coroutine with itertools.cycle
But this is no different with synchronous, so lemme know if you need is asynchronous.
import asyncio
import itertools
import httpx
async def main_task(client: httpx.AsyncClient):
resp = await client.get("http://127.0.0.1:5000/")
print(resp.read())
await asyncio.sleep(0.1) # without this would be IP ban
async def main():
async with httpx.AsyncClient() as client:
for coroutine in itertools.cycle([main_task]):
await coroutine(client)
asyncio.run(main())
Server:
import time
from flask import Flask
app = Flask(__name__)
#app.route("/")
def hello():
return str(time.time())
app.run()
Output:
b'1639918937.7694323'
b'1639918937.8804302'
b'1639918937.9914327'
b'1639918938.1014295'
b'1639918938.2124324'
b'1639918938.3204308'
...
asyncio.create_task() works as you describe it. The problem you are having here is that you create an infinite loop here:
async def save_subs_loop():
while True:
asyncio.create_task(print_subs())
time.sleep(0.1) # do not use time.sleep() in async code EVER
save_subs_loop() keeps creating tasks but control is never yielded back to the event loop, because there is no await in there. Try
async def save_subs_loop():
while True:
asyncio.create_task(print_subs())
await asyncio.sleep(0.1) # yield control back to loop to give tasks a chance to actually run
This problem is so common I'm thinking python should raise a RuntimeError if it detects time.sleep() within a coroutine :-)
You might want to try the TaskThread framework
It allows you to add tasks in runtime
Tasks are re-scheduled periodically (like in your while loop up there)
There is a consumer / producer framework built in (parent/child relationships) which you seem to need
disclaimer: I wrote TaskThread out of necessity & it's been a life saver.
I want to make a timer which is started in a normal function, but in the timer function, it should be able to call an async function
I want to do something like this:
startTimer()
while True:
print("e")
def startTimer(waitForSeconds: int):
# Wait for `waitForSeconds`
await myAsyncFunc()
async def myAsyncFunc():
print("in my async func")
Where the while True loop should do its stuff and after waitForSeconds the timer the async function should execute an other async function, but waiting shouldn't block any other actions and doesn't need to be awaited
If something isn't understandable, I'm sorry, I'll try to explain it then
Thanks
If you want to run your synchronous and asynchronous code in parallel, you will need to run one of them in a separate thread. For example:
def sync_code():
while True:
print("e")
async def start_timer(secs):
await asyncio.sleep(secs)
await async_func()
async def main():
asyncio.create_task(start_timer(1))
loop = asyncio.get_event_loop()
# use run_in_executor to run sync code in a separate thread
# while this thread runs the event loop
await loop.run_in_executor(None, sync_code)
asyncio.run(main())
If the above is not acceptable for you (e.g. because it turns the whole program into an asyncio program), you can also run the event loop in a background thread, and submit tasks to it using asyncio.run_coroutine_threadsafe. That approach would allow startTimer to have the signature (and interface) like you wanted it:
def startTimer(waitForSeconds):
loop = asyncio.new_event_loop()
threading.Thread(daemon=True, target=loop.run_forever).start()
async def sleep_and_run():
await asyncio.sleep(waitForSeconds)
await myAsyncFunc()
asyncio.run_coroutine_threadsafe(sleep_and_run(), loop)
async def myAsyncFunc():
print("in my async func")
startTimer(1)
while True:
print("e")
I'm pretty sure that you are familiar with concurent processing, but you didn't show exactly what you want. So if I understand you correctly you want to have 2 processes. First is doing only while True, and the second process is the timer(waits e.g. 5s) and it will call async task. I assume that you are using asyncio according to tags:
import asyncio
async def myAsyncFunc():
print("in my async func")
async def call_after(delay):
await asyncio.sleep(delay)
await myAsyncFunc()
async def while_true():
while True:
await asyncio.sleep(1) # sleep here to avoid to large output
print("e")
async def main():
task1 = asyncio.create_task(
while_true())
task2 = asyncio.create_task(
call_after(5))
# Wait until both tasks are completed (should take
# around 2 seconds.)
await task1
await task2
asyncio.run(main())
I am new to study about asyncio.I don't know how to
describe my question.But here is a minimal example:
import asyncio
async def work():
await asyncio.sleep(3)
async def check_it():
task = asyncio.create_task(work())
await task
while True:
if task.done():
print("Done")
break
print("Trying...")
asyncio.run(check_it())
My idea is very simple:
create a async task in check_it().And await it.
Use a while loop to check whether the task is finished.
If task.done() return True,break the while loop.Then exit the script.
If my question is duplicate, please flag my question.Thanks!
Try asyncio.wait or use asyncio.sleep. Otherwise, your program will output a lot without some pauses.
import asyncio
async def work():
await asyncio.sleep(3)
async def check_it():
task = asyncio.create_task(work())
# "await" block until the task finish. Do not do here.
timeout = 0 # Probably the first timeout is 0
while True:
done, pending = await asyncio.wait({task}, timeout=timeout)
if task in done:
print('Done')
# Do an await here is favourable in case any exception is raised.
await task
break
print('Trying...')
timeout = 1
asyncio.run(check_it())
I'm very inexperienced with asyncio and asynchronous programming in general, so i've been having a hard time trying to use a synchronous websocket module with the async discord.py module. I am trying to setup a bot which constantly listens for websocket messages, and if it receives data, it performs some calculations and sends a message in discord. This should run indefinitely, and the websocket changes its origin routinely. Here is some commented code of what i'm trying to accomplish:
import requests
import websocket
import discord
import asyncio
from time import sleep
client = discord.Client() # Initialize the discord client
class Wrapper: # This just holds some variables so that I can use them without worrying about scope
data = None
first_time = True
is_running = False
socket_url = None
async def init_bot(): # The problem begins here
def on_message(ws, message):
if is_valid(message['data']): # is_valid is just a name for a special comparison I do with the message
Wrapper.data = message['data']
print('Received valid data')
def on_error(ws, error):
print('There was an error connecting to the websocket.')
ws = websocket.WebSocketApp(Wrapper.socket_url,
on_message=on_message,
on_error=on_error)
while True:
ws.run_forever() # I believe this is connected to the discord event loop?
# Using ws.close() here breaks the program when it receives data for the second time
async def start():
await client.wait_until_ready()
def get_response(hq_id, headers):
response = requests.get('my_data_source')
try:
return json.loads(response.text)
except:
return None
print('Starting websocket')
await init_bot()
while True: # This should run forever, but the outer coroutine gets task pending errors
print('Running main loop')
if Wrapper.first_time:
Wrapper.first_time = False
on_login = await client.send_message(Config.public_channel, embed=Config.waiting_embed()) # Sends an embed to a channel
while Wrapper.socket_url is None:
response = get_response(ID, HEADERS)
try:
Wrapper.socket_url = response['socket_url'] # Assume this sets the socket url to a websocket in the form ws://anyhost.com
await client.edit_message(on_login, embed=Config.connect_embed())
Wrapper.is_running = True
await asyncio.sleep(3)
except:
await asyncio.sleep(60) # The response will only sometimes include a proper socket_url, so we wait for one
if Wrapper.is_running:
while Wrapper.data is None: # Is this blocking? I essentially want this while loop to end when we have data from line 18
await asyncio.sleep(1)
if Wrapper.data is not None:
data_message = await client.send_message(Config.public_channel, embed=Wrapper.data[0])
await client.add_reaction(ans_message, '👍')
await client.add_reaction(ans_message, '👎')
if Wrapper.data[1] == True: # Marks the last message, so we want to reset the bot to its initial state
Wrapper.is_running = False
Wrapper.first_time = True
Wrapper.socket_url = None
await asyncio.sleep(100) # Sleep for a period of time in order to make sure the socket url is closed
Wrapper.data = None
await asyncio.sleep(3)
print('While loop ended?')
#client.event
async def on_ready():
print(f'My Bot\n')
client.loop.create_task(start())
client.run('<TOKEN>')
I've tried several variations of the above, but the error I typically get is something along these lines:
File "mybot.py", line 247, in <module>
client.loop.create_task(start())
task: <Task pending coro=<start() running at mybot.py> wait_for=<Future pending cb=[BaseSelectorEventLoop._sock_connect_done(1016)(), <TaskWakeupMethWrapper object at 0x000002545FFFC8B8>()]
You cannot just mix asyncio-aware code, such as discord, with synchronous websockets code. Since nothing is awaited in init_bot, calling await init_bot() completely stops the event loop.
Instead, you need to run the websocket code (init_bot function in your case) in a separate thread and await an appropriate event. For example:
def init_bot(loop, w):
def on_message(ws, message):
w.data = message['data']
loop.call_soon_threadsafe(w.event.set)
# ...
async def start():
# ...
loop = asyncio.get_event_loop()
w = Wrapper()
w.event = asyncio.Event()
threading.Thread(target=lambda: init_bot(loop, w)).start()
# ...
# instead of while Wrapper.data is None ...
await w.event.wait()
# ... process data
w.event.clear()