I'm programming a server in Python3, which takes screenshot and sends it over websockets. I have coroutine for handling connection and I would like to create another coroutine for taking screenshot at some interval. Screenshot coroutine will probably run in different thread and I will need to propagate the result to some shared variable with read-write lock, to be able to send it. My questions: (result should be multiplatform, if possible)
How is it possible to schedule tasks like this? I created server which runs forever, and I can create periodical coroutine, but somehow I can't put them together in one loop.
What is a good way to propagate the result from one thread (or coroutine, if server is single threaded) to another?
I found this piece of code similar to this and I can't get it to work (second coroutine doesn't execute). Can someone correct this with and without multithreading?
async def print_var():
global number
await asyncio.sleep(2)
print(number)
async def inc_var():
global number
await asyncio.sleep(5)
number += 1
number = 0
asyncio.get_event_loop().run_until_complete(print_var())
asyncio.async(inc_var)
asyncio.get_event_loop().run_forever()
Post-answer edit
In the end after more hours of googling, I actually got it to work on a single thread, so there's no danger of race condition. (But I'm still not sure what ensure_future does, and why it isn't called on event loop.)
users = set()
def register(websocket):
users.add(websocket)
def unregister(websocket):
users.remove(websocket)
async def get_screenshot():
global screenshot
while True:
screenshot = screenshot()
await asyncio.sleep(0.2)
async def server(websocket, path):
global screenshot
register(websocket)
try:
async for message in websocket:
respond(screenshot)
finally:
unregister(websocket)
def main():
asyncio.get_event_loop().run_until_complete(
websockets.serve(server, 'localhost', 6789))
asyncio.ensure_future(get_screenshot())
asyncio.get_event_loop().run_forever()
main()
In Python 3.7:
import asyncio
import websockets
CAPTURE_INTERVAL = 1
running = True
queues = set()
async def handle(ws, path):
queue = asyncio.Queue()
queues.add(queue)
while running:
data = await queue.get()
if not data:
break
await ws.send(data)
def capture_screen():
# Do some work here, preferably in C extension without holding the GIL
return b'screenshot data'
async def main():
global running
loop = asyncio.get_running_loop()
server = await websockets.serve(handle, 'localhost', 8765)
try:
while running:
data = await loop.run_in_executor(None, capture_screen)
for queue in queues:
queue.put_nowait(data)
await asyncio.sleep(CAPTURE_INTERVAL)
finally:
running = False
for queue in queues:
queue.put_nowait(None)
server.close()
await server.wait_closed()
if __name__ == '__main__':
asyncio.run(main())
Please note, this is only for demonstrating the producer-consumer fan-out pattern. The queues are not essential - you can simply send data to all server.sockets in main() directly, while in handle() you should worry about incoming websocket messages. For example, client may control image compression rate like this:
import asyncio
import websockets
CAPTURE_INTERVAL = 1
DEFAULT = b'default'
qualities = {}
async def handle(ws, path):
try:
async for req in ws:
qualities[ws] = req
finally:
qualities.pop(ws, None)
def capture_screen():
# Do some work here, preferably in C extension without holding the GIL
return {
DEFAULT: b'default screenshot data',
b'60': b'data at 60% quality',
b'80': b'data at 80% quality',
}
async def main():
loop = asyncio.get_running_loop()
server = await websockets.serve(handle, 'localhost', 8765)
try:
while True:
data = await loop.run_in_executor(None, capture_screen)
for ws in server.sockets:
quality = qualities.get(ws, DEFAULT)
if quality not in data:
quality = DEFAULT
asyncio.create_task(ws.send(data[quality]))
await asyncio.sleep(CAPTURE_INTERVAL)
finally:
server.close()
await server.wait_closed()
if __name__ == '__main__':
asyncio.run(main())
Related
Based on the solution that i got: Running multiple sockets using asyncio in python
i tried to add also the computation part using asyncio
Setup: Python 3.7.4
import msgpack
import threading
import os
import asyncio
import concurrent.futures
import functools
import nest_asyncio
nest_asyncio.apply()
class ThreadSafeElem(bytes):
def __init__(self, * p_arg, ** n_arg):
self._lock = threading.Lock()
def __enter__(self):
self._lock.acquire()
return self
def __exit__(self, type, value, traceback):
self._lock.release()
elem = ThreadSafeElem()
async def serialize(data):
return msgpack.packb(data, use_bin_type=True)
async def serialize1(data1):
return msgpack.packb(data1, use_bin_type=True)
async def process_data(data,data1):
loop = asyncio.get_event_loop()
future = await loop.run_in_executor(None, functools.partial(serialize, data))
future1 = await loop.run_in_executor(None, functools.partial(serialize1, data1))
return await asyncio.gather(future,future1)
################ Calculation#############################
def calculate_data():
global elem
while True:
try:
... data is calculated (some dictionary))...
elem, elem1= asyncio.run(process_data(data, data1))
except:
pass
#####################################################################
def get_data():
return elem
def get_data1():
return elem1
########### START SERVER AND get data contionusly ################
async def client_thread(reader, writer):
while True:
try:
bytes_received = await reader.read(100)
package_type = np.frombuffer(bytes_received, dtype=np.int8)
if package_type ==1 :
nn_output = get_data1()
if package_type ==2 :
nn_output = get_data()
writer.write(nn_output)
await writer.drain()
except:
pass
async def start_servers(host, port):
server = await asyncio.start_server(client_thread, host, port)
await server.serve_forever()
async def start_calculate():
await asyncio.run(calculate_data())
def enable_sockets():
try:
host = '127.0.0.1'
port = 60000
sockets_number = 6
loop = asyncio.get_event_loop()
for i in range(sockets_number):
loop.create_task(start_servers(host,port+i))
loop.create_task(start_calculate())
loop.run_forever()
except:
print("weird exceptions")
##############################################################################
enable_sockets()
The issue is that when i make a call from client, the server does not give me anything.
I tested the program with dummy data and no asyncio on calculation part so without this loop.create_task(start_calculate()) and the server responded correctly.
I also run the calculate data without adding it in the enable sockets and it worked. It also working with this implementation, but the problem is the server is not returning anything.
I did it like this cos i need the calculate part to run continuously and when one of the clients is calling to return the data at that point.
An asyncio event loop cannot be nested inside another, and there is no point in doing so: asyncio.run (and similar) blocks the current thread until done. This does not increase parallelism, and merely disables any outer event loop.
If you want to nest another asyncio task, directly run it in the current event loop. If you want to run a non-cooperative, blocking task, run it in the event loop executor.
async def start_calculate():
loop = asyncio.get_running_loop()
await loop.run_in_executor(None, calculate_data)
The default executor uses threads – this allows running blocking tasks, but does not increase parallelism. Use a custom ProcessPoolExecutor to use additional cores:
import concurrent.futures
async def start_calculate():
loop = asyncio.get_running_loop()
with concurrent.futures.ProcessPoolExecutor() as pool:
await loop.run_in_executor(pool, calculate_data)
Why do you call asyncio.run() multiple times?
This function always creates a new event loop and closes it at the end. It should be used as a main entry point for asyncio programs, and should ideally >only be called once.
I would advise you to read the docs
I am trying to create a script in python that listens to multiple sockets using websockets and asyncio, the problem is that no matter what I do it only listen to the first socket I call.
I think its the infinite loop, what are my option to solve this? using threads for each sockets?
async def start_socket(self, event):
payload = json.dumps(event)
loop = asyncio.get_event_loop()
self.tasks.append(loop.create_task(
self.subscribe(event)))
# this should not block the rest of the code
await asyncio.gather(*tasks)
def test(self):
# I want to be able to add corotines at a different time
self.start_socket(event1)
# some code
self.start_socket(event2)
this is what I did eventually, that way its not blocking the main thread and all subscriptions are working in parallel.
def subscribe(self, payload):
ws = websocket.WebSocket(sslopt={"cert_reqs": ssl.CERT_NONE})
ws.connect(url)
ws.send(payload)
while True:
result = ws.recv()
print("Received '%s'" % result)
def start_thread(self, loop):
asyncio.set_event_loop(loop)
loop.run_forever()
def start_socket(self, **kwargs):
worker_loop = asyncio.new_event_loop()
worker = Thread(target=self.start_thread, args=(worker_loop,))
worker.start()
worker_loop.call_soon_threadsafe(self.subscribe, payload)
def listen(self):
self.start_socket(payload1)
# code
self.start_socket(payload2)
# code
self.start_socket(payload3)
Your code appears incomplete, but what you've shown has two issues. One is that run_until_complete accepts a coroutine object (or other kind of future), not a coroutine function. So it should be:
# note parentheses after your_async_function()
asyncio.get_event_loop().run_until_complete(your_async_function())
the problem is that no matter what I do it only listen to the first socket I call. I think its the infinite loop, what are my option to solve this? using threads for each sockets?
The infinite loop is not the problem, asyncio is designed to support such "infinite loops". The problem is that you are trying to do everything in one coroutine, whereas you should be creating one coroutine per websocket. This is not a problem, as coroutines are very lightweight.
For example (untested):
async def subscribe_all(self, payload):
loop = asyncio.get_event_loop()
# create a task for each URL
for url in url_list:
tasks.append(loop.create_task(self.subscribe_one(url, payload)))
# run all tasks in parallel
await asyncio.gather(*tasks)
async def subsribe_one(self, url, payload):
async with websockets.connect(url) as websocket:
await websocket.send(payload)
while True:
msg = await websocket.recv()
print(msg)
One way to efficiently listen to multiple websocket connections from a websocket server is to keep a list of connected clients and essentially juggle multiple conversations in parallel.
E.g. A simple server that sends random # to each connected client every few secs:
import os
import asyncio
import websockets
import random
websocket_clients = set()
async def handle_socket_connection(websocket, path):
"""Handles the whole lifecycle of each client's websocket connection."""
websocket_clients.add(websocket)
print(f'New connection from: {websocket.remote_address} ({len(websocket_clients)} total)')
try:
# This loop will keep listening on the socket until its closed.
async for raw_message in websocket:
print(f'Got: [{raw_message}] from socket [{id(websocket)}]')
except websockets.exceptions.ConnectionClosedError as cce:
pass
finally:
print(f'Disconnected from socket [{id(websocket)}]...')
websocket_clients.remove(websocket)
async def broadcast_random_number(loop):
"""Keeps sending a random # to each connected websocket client"""
while True:
for c in websocket_clients:
num = str(random.randint(10, 99))
print(f'Sending [{num}] to socket [{id(c)}]')
await c.send(num)
await asyncio.sleep(2)
if __name__ == "__main__":
loop = asyncio.get_event_loop()
try:
socket_server = websockets.serve(handle_socket_connection, 'localhost', 6789)
print(f'Started socket server: {socket_server} ...')
loop.run_until_complete(socket_server)
loop.run_until_complete(broadcast_random_number(loop))
loop.run_forever()
finally:
loop.close()
print(f"Successfully shutdown [{loop}].")
A simple client that connects to the server and listens for the numbers:
import asyncio
import random
import websockets
async def handle_message():
uri = "ws://localhost:6789"
async with websockets.connect(uri) as websocket:
msg = 'Please send me a number...'
print(f'Sending [{msg}] to [{websocket}]')
await websocket.send(msg)
while True:
got_back = await websocket.recv()
print(f"Got: {got_back}")
asyncio.get_event_loop().run_until_complete(handle_message())
Mixing up threads and asyncio is more trouble than its worth and you still have code that will block on the most wasteful steps like network IO (which is the essential benefit of using asyncio).
You need to run each coroutine asynchronously in an event loop, call any blocking calls with await and define each method that interacts with any awaitable interactions with an async
See a working e.g.: https://github.com/adnantium/websocket_client_server
I'm very inexperienced with asyncio and asynchronous programming in general, so i've been having a hard time trying to use a synchronous websocket module with the async discord.py module. I am trying to setup a bot which constantly listens for websocket messages, and if it receives data, it performs some calculations and sends a message in discord. This should run indefinitely, and the websocket changes its origin routinely. Here is some commented code of what i'm trying to accomplish:
import requests
import websocket
import discord
import asyncio
from time import sleep
client = discord.Client() # Initialize the discord client
class Wrapper: # This just holds some variables so that I can use them without worrying about scope
data = None
first_time = True
is_running = False
socket_url = None
async def init_bot(): # The problem begins here
def on_message(ws, message):
if is_valid(message['data']): # is_valid is just a name for a special comparison I do with the message
Wrapper.data = message['data']
print('Received valid data')
def on_error(ws, error):
print('There was an error connecting to the websocket.')
ws = websocket.WebSocketApp(Wrapper.socket_url,
on_message=on_message,
on_error=on_error)
while True:
ws.run_forever() # I believe this is connected to the discord event loop?
# Using ws.close() here breaks the program when it receives data for the second time
async def start():
await client.wait_until_ready()
def get_response(hq_id, headers):
response = requests.get('my_data_source')
try:
return json.loads(response.text)
except:
return None
print('Starting websocket')
await init_bot()
while True: # This should run forever, but the outer coroutine gets task pending errors
print('Running main loop')
if Wrapper.first_time:
Wrapper.first_time = False
on_login = await client.send_message(Config.public_channel, embed=Config.waiting_embed()) # Sends an embed to a channel
while Wrapper.socket_url is None:
response = get_response(ID, HEADERS)
try:
Wrapper.socket_url = response['socket_url'] # Assume this sets the socket url to a websocket in the form ws://anyhost.com
await client.edit_message(on_login, embed=Config.connect_embed())
Wrapper.is_running = True
await asyncio.sleep(3)
except:
await asyncio.sleep(60) # The response will only sometimes include a proper socket_url, so we wait for one
if Wrapper.is_running:
while Wrapper.data is None: # Is this blocking? I essentially want this while loop to end when we have data from line 18
await asyncio.sleep(1)
if Wrapper.data is not None:
data_message = await client.send_message(Config.public_channel, embed=Wrapper.data[0])
await client.add_reaction(ans_message, '👍')
await client.add_reaction(ans_message, '👎')
if Wrapper.data[1] == True: # Marks the last message, so we want to reset the bot to its initial state
Wrapper.is_running = False
Wrapper.first_time = True
Wrapper.socket_url = None
await asyncio.sleep(100) # Sleep for a period of time in order to make sure the socket url is closed
Wrapper.data = None
await asyncio.sleep(3)
print('While loop ended?')
#client.event
async def on_ready():
print(f'My Bot\n')
client.loop.create_task(start())
client.run('<TOKEN>')
I've tried several variations of the above, but the error I typically get is something along these lines:
File "mybot.py", line 247, in <module>
client.loop.create_task(start())
task: <Task pending coro=<start() running at mybot.py> wait_for=<Future pending cb=[BaseSelectorEventLoop._sock_connect_done(1016)(), <TaskWakeupMethWrapper object at 0x000002545FFFC8B8>()]
You cannot just mix asyncio-aware code, such as discord, with synchronous websockets code. Since nothing is awaited in init_bot, calling await init_bot() completely stops the event loop.
Instead, you need to run the websocket code (init_bot function in your case) in a separate thread and await an appropriate event. For example:
def init_bot(loop, w):
def on_message(ws, message):
w.data = message['data']
loop.call_soon_threadsafe(w.event.set)
# ...
async def start():
# ...
loop = asyncio.get_event_loop()
w = Wrapper()
w.event = asyncio.Event()
threading.Thread(target=lambda: init_bot(loop, w)).start()
# ...
# instead of while Wrapper.data is None ...
await w.event.wait()
# ... process data
w.event.clear()
I have a python 3.6 program where I am using the asyncio package event loops. One of my data sources comes from an api which was not build around asyncio. My connection object contains a member called _connection which is just a python socket. Right now I can use this in a select statement to tell when data is ready.
async def run(self):
while True:
if select.select([self._q._connection], [], [])[0]:
msg = self._q.receive()
print(msg)
What I would really like is...
async def run(self):
while True:
if await select.select([self._q._connection], [], [])[0]:
msg = self._q.receive()
print(msg)
I know there is a sock_recv function in the asyncio event loop however I need the api to do the actual reading and decoding. I tried this but it would just fall through the await which I guess makes sense since I said 0 bytes.
async def run(self):
while True:
print('A')
await asyncio.get_event_loop().sock_recv(self._q._connection, 0)
print('B')
msg = self._q.receive()
print(msg)
The only solution I can think of for now is to add a small timeout to the select and then call asyncio.sleep while there is no data but this seems like an inefficent approach. I wish there was something like asyncio.select. Do anyone want to recommend another approach?
EDIT: Right now I have come up with this. I don't like it because it adds an extra quarter second latency (probably doesn't matter much for my application but it still bugs me.)
async def run(self):
while True:
if select.select([self._q._connection], [], [], 0)[0]:
print(self._q.receive())
else:
await asyncio.sleep(0.25)
You could use loop.add_reader to wait for the read availability of your socket:
async def watch(fd):
future = asyncio.Future()
loop.add_reader(fd, future.set_result, None)
future.add_done_callback(lambda f: loop.remove_reader(fd))
await future
async def run(self):
while True:
await watch(self._q._connection)
msg = self._q.receive()
print(msg)
However, it'll be very tricky to avoid all the blocking IO calls of the library you mentioned without rewriting it completely. Instead, I'd recommend to use the loop.run_in_executor method to schedule the blocking IO calls in a thread pool:
async def run(self):
loop = asyncio.get_event_loop()
while True:
msg = await loop.run_in_executor(None, self._q.receive)
print(msg)
I am currently developing a server program in Python that uses the websockets and asyncio packages.
I got a basic script handling websockets working (Exhibit A). This script locks when waiting for input, which is not what I want.
The solution for this that I imagine is I can start two asynchronous tasks - one that handles inputs and one that handles outputs - and start them in a secondary event loop. I had to do some research about coroutines, and I came up with Exhibit B as a proof of concept for running two things simultaneously in an event loop.
Now what I'm stuck on is Exhibit C. When I attempted to use this in a practical scenario with the websockets package, I found that websocket.recv() never finishes (or the coroutine never un-pauses - I'm not sure what's going on exactly). In exhibit A it works fine, and I've determined that the coroutine definitely runs at least up until that point.
Any ideas?
Exhibit A:
#!/usr/bin/python3
import asyncio
import websockets
import time
# This works great!
async def hello(websocket, path):
while True:
# This line waits for input from socket
name = await websocket.recv()
print("< {}".format(name))
# "echo... echo... echo... echo... echo..."
greeting = ''.join(name + "... " for x in range(5))
await websocket.send(greeting)
print("> {}".format(greeting))
time.sleep(0.1);
start_server = websockets.serve(hello, '', 26231)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()
Exhibit B:
#!/usr/bin/python3
import asyncio
import time
class Thing:
def __init__(self):
pass
async def test(self):
for x in range(10):
print("Hello"+str(x))
await asyncio.sleep(0)
def run(self):
# Add the task to the event loop twice
asyncio.ensure_future(self.test())
asyncio.ensure_future(self.test())
t = Thing()
t.run()
loop = asyncio.get_event_loop();
loop.run_forever()
Exhibit C:
#!/usr/bin/python3
import asyncio
import websockets
import time
class WebsocketRequest:
def __init__(self, websocket):
self.websocket = websocket
# Works great
async def handle_oputs(self):
# This works fine - sends a message
# every 10 seconds to the client
while True:
print("sending...")
await self.websocket.send("Hello")
print("> {}".format("Hello"))
time.sleep(10)
# Doesn't work
async def handle_iputs(self):
# This stops at the await and never receives
# any input from the client :/
while True:
try:
print("receiving...")
# This is the line that causes sadness
data = await self.websocket.recv()
print("< {}".format(data))
except:
# This doesn't happen either
print("Listener is dead")
async def run(self):
# This is the part where the coroutine for
# a client get split off into two of them
# to handle input and output separately.
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
asyncio.ensure_future(self.handle_iputs())
asyncio.ensure_future(self.handle_oputs())
loop.run_forever()
class WebsocketServer:
def __init__(self, address):
self.ip = address[0]
self.port = address[1]
async def hello(self, websocket, path):
req = WebsocketRequest(websocket)
await req.run()
def run(self):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
start_server = websockets.serve(self.hello, self.ip, self.port)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()
Maybe the module websocket (unlike websockets) can help you.
The use of WebsocketApp is very simple:
from websocket import WebSocketApp
class ExampleClass(object):
def __init__(self):
websocket.enableTrace(True)
self.ws = websocket.WebSocketApp("ws://echo.websocket.org",
on_message=on_message,
on_error=on_error,
on_close=on_close)
def on_message(ws, msg):
print "Message Arrived:" + msg
def on_error(ws, error):
print error
def on_close(ws):
print "Connection Closed"
def on_open(ws):
ws.send("Hello!")
To download this module: https://pypi.python.org/pypi/websocket-client
time.sleep() is a blocking operation, so any other tasks cannot interrupt and are not scheduled. Use await asyncio.sleep() instead.