my code:
import asyncio
async def count(counter):
print(f"number of entries in the list {len(counter)}")
while True:
await asyncio.sleep(1 / 1000)
counter.append(1)
async def print_every_sec(counter):
while True:
await asyncio.sleep(1)
print(f"- 1 secund later. " f"number of entries in the list: {len(counter)}")
async def print_every_5_sec():
while True:
await asyncio.sleep(5)
print(f"---- 5 secund later")
async def print_every_10_sec():
while True:
await asyncio.sleep(10)
print(f"---------- 10 secund later")
async def main():
counter = list()
tasks = [
count(counter),
print_every_sec(counter),
print_every_5_sec(),
print_every_10_sec(),
]
await asyncio.gather(*tasks)
asyncio.run(main())
This is my conclusion but is not correct.
Correct conclusion around 1000 for each iteration.
I don't now what is it. This code works fine in online interpretations.
The assumption that asyncio.sleep(1 / 1000) (and to return control to other async routines) takes exactly one millisecond is not true.
Here's a more interesting example that records how long the sleep (and the time.perf_counter_ns() invocation) actually took:
import asyncio
import statistics
import time
max_count = 2500
async def count(counter):
while len(counter) < max_count:
t0 = time.perf_counter_ns()
await asyncio.sleep(1 / 1000)
t1 = time.perf_counter_ns()
counter.append((t1 - t0) / 1_000_000)
async def print_every_sec(counter):
while len(counter) < max_count:
await asyncio.sleep(1)
print(f'count: {len(counter)}; average: {statistics.mean(counter)} ms')
async def main():
counter = list()
tasks = [
count(counter),
print_every_sec(counter),
]
await asyncio.gather(*tasks)
asyncio.run(main())
On my Macbook, Python 3.9, etc, etc., the result is
count: 744; average: 1.341670
count: 1494; average: 1.33668
count: 2248; average: 1.33304
count: 2500; average: 1.325463428
so it takes 30% more than we expected to.
For sleeps of 10ms, the average is 11.84 ms. For sleeps of 100ms, the average is 102.9 ms.
Related
I am creating a python code that does the following:
1 - creates an asynchronous function called "nums" than prints random numbers from 0 to 10 every 2 seconds.
2 - execute a function that prints "hello world" every 5 seconds.
3 - if the function "nums" prints 1 or 3, break the loop and end the program.
The thing is, it's not breaking out of the loop. What I am supposed to do? Thank you.
import asyncio
import random
async def nums():
while True:
x = print(random.randint(0, 10))
await asyncio.sleep(2)
if x == 1 or x == 3:
break
async def world():
while True:
print("hello world")
await asyncio.sleep(5)
async def main():
while True:
await asyncio.wait([nums(), world()])
if __name__ == '__main__':
asyncio.run(main())
You can use return_when=asyncio.FIRST_COMPLETED parameter in asyncio.wait:
import asyncio
import random
async def nums():
while True:
x = random.randint(0, 10)
print(x)
if x in {1, 3}
break
await asyncio.sleep(2)
async def world():
while True:
print("hello world")
await asyncio.sleep(5)
async def main():
task1 = asyncio.create_task(nums())
task2 = asyncio.create_task(world())
await asyncio.wait({task1, task2}, return_when=asyncio.FIRST_COMPLETED)
if __name__ == "__main__":
asyncio.run(main())
Prints (for example):
5
hello world
6
6
hello world
1
<program ends here>
I am writing a personal Telegram Bot to collect statistics on my accounts market.csgo.com. The main task of the bot is to send async requests to the API and display information via Telegram. Everything works as it should, but the problem is with global variables, or rather, their incorrect counting. An example of one of my functions:
...
import asyncio
import aiohttp
sale_total_sum = 0
amount_total_items = 0
async def get_on_sale(session, dictt, message):
global sale_total_sum
global amount_total_items
async with session.get(f'https://market.csgo.com/api/v2/items?key={dictt[1][1]}') as resp:
html = await resp.json()
if html['items'] is None:
pass
else:
each_sale_sum = 0
each_amount_items = 0
for i in html['items']:
sale_total_sum += i['price']
each_sale_sum += i['price']
each_amount_items += 1
amount_total_items += 1
try:
await bot.send_message(message.from_user.id,
f'{dictt[0]} : <b>{each_sale_sum} $</b>\nItems: <i>{each_amount_items}</i>',
disable_web_page_preview=True, parse_mode=types.ParseMode.HTML)
except exceptions.RetryAfter as e:
await asyncio.sleep(e.timeout)
#dp.message_handler(content_types=['text'])
async def Main(message):
profiles = users()
async with aiohttp.ClientSession(trust_env=True) as session:
tasks = []
if message.text == 'On Sale 💰':
await bot.send_message(message.from_user.id, 'Information request. Wait..')
for i in profiles.items():
task = asyncio.ensure_future(get_on_sale(session, i, message))
tasks.append(task)
await asyncio.gather(*tasks)
await bot.send_message(message.from_user.id,
f'<b>Total on sale: {sale_total_sum} $\nTotal items: {amount_total_items}\nBot start at: {start}</b>',
reply_markup=kb_client, parse_mode=types.ParseMode.HTML)
executor.start_polling(dp, skip_updates=True)
Function result:
Account_1: 100 $
Items: 1
Account_2: 200 $
Items: 2
Account_3: 300 $
Items: 3
Total on sale: 600 $
Total items: 6
Bot works in polling mode executor.start_polling(dp, skip_updates=True). If the function async def get_on_sale is called for the first time after enabling, then its final count Total on sale: 600 ₽ Total items: 6 will be correct, but subsequent calls will double this amount, which is actually not like this:
Account_1: 100 $
Items: 1
Account_2: 200 $
Items: 2
Account_3: 300 $
Items: 3
Total on sale: 1200 $
Total items: 12
I know that the problem is in the global variables global sale_total_sum and global amount_total_items. But if you use simple variables instead, they will simply be overwritten, and not summarized as I need. Therefore, I want to ask - is there a way to somehow reset or reassign these global variables to 0 after the function ends? So that the data would be correct on the next call. Thank you.
The solution was to create an object with two fields where get_on_sale is the argument
from dataclasses import dataclass
#dataclass
class AggregatedStats:
sum: int = 0
items: int = 0
async def get_on_sale(session, dictt, message, stats):
...
for i in html['items']:
stats.sum += i['price']
each_sale_sum += i['price']
each_amount_items += 1
stats.items += 1
...
#dp.message_handler(content_types=['text'])
async def Main(message):
profiles = users()
async with aiohttp.ClientSession(trust_env=True) as session:
tasks = []
if message.text == 'On Sale 💰':
await bot.send_message(message.from_user.id, 'Information request. Wait..')
stats = AggregatedStats()
for i in profiles.items():
task = asyncio.ensure_future(get_on_sale(session, i, message, stats))
tasks.append(task)
await asyncio.gather(*tasks)
await bot.send_message(message.from_user.id,
f'<b>Total on sale: {stats.sum} ₽\nTotal items: {stats.items}\nBot start at: {start}</b>',
reply_markup=kb_client, parse_mode=types.ParseMode.HTML)
I have an async method, as shown below.
I pass in lists of 1000 numbers, where the method will pass in each number to a helper function which will return something from a website.
I have a global variable called count, which i surround with locks to make sure it doesnt get changed by anything else
I use add_done_callback with the task to make this method async.
The goal is to keep sending a number in the list of 1000 numbers to the server, and only when the server returns data (can take anywhere from 0.1 to 2 seconds), to pause, write the data to a sql database, and then continue
The code works as expected without locks, or without making the callback function, (which is named 'function' below) asyncrounous. But adding locks gives me an error: RuntimeWarning: coroutine 'function' was never awaited self._context.run(self._callback, *self._args) RuntimeWarning: Enable tracemalloc to get the object allocation traceback
I am super new to async in python so any help/advice is greatly appriciated
My code is shown below. It is just a simple draft:
import time
import random
import asyncio
# from helper import get_message_from_server
async def get(number):
# get_message_from_server(number), which takes somewhere between 0.1 to 2 seconds
await asyncio.sleep(random.uniform(0.1, 2))
s = 'Done with number ' + number
return s
async def function(future, lock):
global count
print(future.result())
# write future.result() to db
acquired = await lock.acquire()
count -= 1 if (count > 1) else 0
lock.release()
async def main(numbers, lock):
global count
count = 0
for i, number in enumerate(numbers):
print('number:', number, 'count:', count)
acquired = await lock.acquire()
count += 1
lock.release()
task = asyncio.create_task(get(number))
task.add_done_callback(
lambda x: function(x, lock)
)
if (count == 50):
print('Reached 50')
await task
acquired = await lock.acquire()
count = 0
lock.release()
if (i == len(numbers) - 1):
await task
def make_numbers():
count = []
for i in range(1001):
count.append(str(i))
return count
if __name__ == '__main__':
numbers = make_numbers()
loop = asyncio.get_event_loop()
lock = asyncio.Lock()
try:
loop.run_until_complete(main(numbers, lock))
except Exception as e:
pass
finally:
loop.run_until_complete(loop.shutdown_asyncgens())
loop.stop()
The above comment helped a lot
This is what the final working code looks like:
import time
import random
import asyncio
from functools import partial
# from helper import get_message_from_server
async def get(number):
# get_message_from_server(number), which takes somewhere between 0.1 to 2 seconds
await asyncio.sleep(random.uniform(0.1, 2))
s = 'Done with number ' + number
return s
def function(result, lock):
print(result.result())
async def count_decrement(lock):
global count
print('in count decrement')
acquired = await lock.acquire()
count -= 1 if (count > 1) else 0
lock.release()
asyncio.create_task(count_decrement(lock))
async def main(numbers, lock):
global count
count = 0
for i, number in enumerate(numbers):
print('number:', number, 'count:', count)
acquired = await lock.acquire()
count += 1
lock.release()
task = asyncio.create_task(get(number))
task.add_done_callback(partial(function, lock = lock))
if (count == 50):
print('Reached 50')
await task
acquired = await lock.acquire()
count = 0
lock.release()
if (i == len(numbers) - 1):
await task
def make_numbers():
count = []
for i in range(1001):
count.append(str(i))
return count
if __name__ == '__main__':
numbers = make_numbers()
loop = asyncio.get_event_loop()
lock = asyncio.Lock()
try:
loop.run_until_complete(main(numbers, lock))
except Exception as e:
pass
finally:
loop.run_until_complete(loop.shutdown_asyncgens())
loop.stop()
I am writing a minimalist code based on asyncio and composed of tasks, workers and a queue:
Workers look in the queue for a task and execute it
Tasks are represented as a random asyncio.sleep()
When a task is done it adds two more tasks to the queue
Two restrictions:
10 max worker at a time
100 tasks should be executed in total, the script should end then
To monitor the total number of tasks executed, I use a global variable task_queued updated each time a task is being added to the queue.
I'm sure there's a better, pythonic way, to do that and not using a global variable, but all the solution I came up with are much more complicated.
I'm missing something here, any clue ?
Here's my code:
import asyncio
from random import random
import sys
MAX_WORKERS = 10
MAX_TASKS = 100
task_queued = 0
async def task(queue, id="1"):
global task_queued
sleep_time = 0.5 + random()
print(' Begin task #{}'.format(id))
await asyncio.sleep(sleep_time)
if task_queued < MAX_TASKS:
await queue.put(id + ".1")
task_queued += 1
if task_queued < MAX_TASKS:
await queue.put(id + ".2")
task_queued += 1
print(' End task #{} ({} item(s) in the queue)'.format(id, queue.qsize()))
async def worker(worker_id, queue):
while True:
task_id = await queue.get()
print('Worker #{} takes charge of task {}'.format(worker_id, task_id))
await task(queue, task_id)
queue.task_done()
async def main():
global task_queued
print('Begin main \n')
queue = asyncio.Queue()
await queue.put("1") # We add one task to the queue
task_queued += 1
workers = [asyncio.create_task((worker(worker_id + 1, queue))) for worker_id in range(MAX_WORKERS)]
await queue.join()
print('Queue is empty, {} tasks completed'.format(task_queued))
for w in workers:
w.cancel()
print('\n End main')
if __name__ == '__main__':
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(main())
except KeyboardInterrupt:
print('\nBye bye')
sys.exit(0)
Thank to user4815162342 for the answer, here's the code if anyone is interested
import asyncio
from random import random
import sys
class Factory:
"""
Factory
"""
def __init__(self, max_workers, max_tasks):
self.task_queued = 0
self.max_workers = max_workers
self.max_tasks = max_tasks
self.queue = asyncio.Queue()
async def task(self, task_id):
sleep_time = 0.5 + random()
print(' Begin task #{}'.format(task_id))
await asyncio.sleep(sleep_time)
if self.task_queued < self.max_tasks:
await self.queue.put(task_id + ".1")
self.task_queued += 1
if self.task_queued < self.max_tasks:
await self.queue.put(task_id + ".2")
self.task_queued += 1
print(' End task #{} ({} item(s) in the queue)'.format(task_id, self.queue.qsize()))
async def worker(self, worker_id):
while True:
task_id = await self.queue.get()
print('Worker #{} takes charge of task {}'.format(worker_id, task_id))
await self.task(task_id)
self.queue.task_done()
async def organize_work(self):
print('Begin work \n')
await self.queue.put("1") # We add one task to the queue to start
self.task_queued += 1
workers = [asyncio.create_task((self.worker(worker_id + 1))) for worker_id in range(self.max_workers)]
await self.queue.join()
print('Queue is empty, {} tasks completed'.format(self.task_queued))
for w in workers:
w.cancel()
print('\nEnd work')
if __name__ == '__main__':
loop = asyncio.get_event_loop()
factory = Factory(max_workers=3, max_tasks=50)
try:
loop.run_until_complete(factory.organize_work())
except KeyboardInterrupt:
print('\nBye bye')
sys.exit(0)
I have this command:
if message.content.lower().startswith("!activate"):
if message.author.id not in tempo:
await Bot.send_message(message.channel, "{} used the command".format(message.author.name))
tempo.append(message.author.id)
await asyncio.sleep(720)
tempo.remove(message.author.id)
else:
await Bot.send_message(message.channel, "wait {} hours.".format(asyncio.sleep))
I would like every time the person tried to use the command a second time, show how much time is left before they can use it again, something like: "wait 4 hours and 50 minutes."
what should I do?
You can just store the time and calculate the remaining yourself:
import time
tempo = {} # use a dict to store the time
if message.content.lower().startswith("!activate"):
if message.author.id not in tempo:
await Bot.send_message(message.channel, "{} used the command".format(
message.author.name))
tempo[message.author.id] = time.time() + 720 # store end time
await asyncio.sleep(720)
del tempo[message.author.id]
else:
await Bot.send_message(message.channel, "wait {} seconds.".format(
tempo[message.author.id] - time.time()))