We Keep Coding

Best way to handle 2 websocket connections in the same time

i am handling data from 2 websocket servers and i would like to know whats the fastest way to handle both connections in the same time given that the 1st connection would send data every 0.1-10ms.
what i am doing so far is:
import json
import websockets
async def run():
async with websockets.connect("ws://localhost:8546/") as ws1:
async with websockets.connect(uri="wss://api.blxrbdn.com/ws", extra_headers = {"Authorization": "apikey") as ws2:
sub1 = await ws1.send("subscription 1")
sub2 = await ws2.send("subscription 2")
while True:
try:
msg1 = await ws1.recv()
msg1 = json.loads(msg1)
msg2 = await ws2.recv()
msg2 = json.loads(msg2)
# process msg1 & msg2
except Exception as e:
print(e, flush=True)
asyncio.run(run())

As stated in the comments, try to handle each connection in its own coroutine. Here is small example:
import asyncio
import websockets
async def worker(ws, msg, t):
while True:
sub = await ws.send(msg)
print("Received from the server:", await ws.recv())
await asyncio.sleep(t)
async def run():
url1 = "ws://localhost:8765/"
url2 = "ws://something_different:8765/"
async with websockets.connect(url1) as ws1, websockets.connect(url2) as ws2:
await asyncio.gather(worker(ws1, "sub1", 1), worker(ws2, "sub2", 2))
asyncio.run(run())

Correct destruction process for async code running in a thread

Below is (working) code for a generic websocket streamer.
It creates a daemon thread from which performs asyncio.run(...).
The asyncio code spawns 2 tasks, which never complete.
How to correctly destroy this object?
One of the tasks is executing a keepalive 'ping', so I can easily exit that loop using a flag. But the other is blocking on a message from the websocket.
import json
import aiohttp
import asyncio
import gzip
import asyncio
from threading import Thread
class WebSocket:
KEEPALIVE_INTERVAL_S = 10
def __init__(self, url, on_connect, on_msg):
self.url = url
self.on_connect = on_connect
self.on_msg = on_msg
self.streams = {}
self.worker_thread = Thread(name='WebSocket', target=self.thread_func, daemon=True).start()
def thread_func(self):
asyncio.run(self.aio_run())
async def aio_run(self):
async with aiohttp.ClientSession() as session:
self.ws = await session.ws_connect(self.url)
await self.on_connect(self)
async def ping():
while True:
print('KEEPALIVE')
await self.ws.ping()
await asyncio.sleep(WebSocket.KEEPALIVE_INTERVAL_S)
async def main_loop():
async for msg in self.ws:
def extract_data(msg):
if msg.type == aiohttp.WSMsgType.BINARY:
as_bytes = gzip.decompress(msg.data)
as_string = as_bytes.decode('utf8')
as_json = json.loads(as_string)
return as_json
elif msg.type == aiohttp.WSMsgType.TEXT:
return json.loads(msg.data)
elif msg.type == aiohttp.WSMsgType.ERROR:
print('⛔️ aiohttp.WSMsgType.ERROR')
return msg.data
data = extract_data(msg)
self.on_msg(data)
# May want this approach if we want to handle graceful shutdown
# W.task_ping = asyncio.create_task(ping())
# W.task_main_loop = asyncio.create_task(main_loop())
await asyncio.gather(
ping(),
main_loop()
)
async def send_json(self, J):
await self.ws.send_json(J)

I'd suggest the use of asyncio.run_coroutine_threadsafe instead of asyncio.run. It returns a concurrent.futures.Future object which you can cancel:
def thread_func(self):
self.future = asyncio.run_coroutine_threadsafe(
self.aio_run(),
asyncio.get_event_loop()
)
# somewhere else
self.future.cancel()
Another approach would be to make ping and main_loop a task, and cancel them when necessary:
# inside `aio_run`
self.task_ping = asyncio.create_task(ping())
self.main_loop_task = asyncio.create_task(main_loop())
await asyncio.gather(
self.task_ping,
self.main_loop_task
return_exceptions=True
)
# somewhere else
self.task_ping.cancel()
self.main_loop_task.cancel()
This doesn't change the fact that aio_run should also be called with asyncio.run_coroutine_threadsafe. asyncio.run should be used as a main entry point for asyncio programs and should be only called once.

I would like to suggest one more variation of the solution. When finishing coroutines (tasks), I prefer minimizing the use of cancel() (but not excluding), since sometimes it can make it difficult to debug business logic (keep in mind that asyncio.CancelledError does not inherit from an Exception).
In your case, the code might look like this(only changes):
class WebSocket:
KEEPALIVE_INTERVAL_S = 10
def __init__(self, url, on_connect, on_msg):
# ...
self.worker_thread = Thread(name='WebSocket', target=self.thread_func)
self.worker_thread.start()
async def aio_run(self):
self._loop = asyncio.get_event_loop()
# ...
self._ping_task = asyncio.create_task(ping())
self._main_task = asyncio.create_task(main_loop())
await asyncio.gather(
self._ping_task,
self._main_task,
return_exceptions=True
)
# ...
async def stop_ping(self):
self._ping_task.cancel()
try:
await self._ping_task
except asyncio.CancelledError:
pass
async def _stop(self):
# wait ping end before socket closing
await self.stop_ping()
# lead to correct exit from `async for msg in self.ws`
await self.ws.close()
def stop(self):
# wait stopping ping and closing socket
asyncio.run_coroutine_threadsafe(
self._stop(), self._loop
).result()
self.worker_thread.join() # wait thread finish

While loop and asyncio

Good day!
I am trying to code a WebSocket connector and using asyncio. I am not that much familiar with asynchronous approaches therefore an incorrect behaviour occurs. Below is the simplified version of the code.
import pandas as pd
import json
import websockets
import asyncio
import time
class BinanceQuotesWS:
def __init__(self,client,pair):
self.quotes = pd.DataFrame(columns=['Timestamp','Price'])
self.pair = pair
self.socket='wss://fstream.binance.com/ws'
self.websocket = None
self.loop = None
self.result = None
def get_quotes(self):
return self.quotes
def start(self):
self.loop = asyncio.get_event_loop()
self.result = self.loop.create_task(self.connect())
async def connect(self):
self.websocket = await websockets.connect(self.socket)
await self.subscribe_quotes()
async def subscribe_quotes(self):
subscribe_message = {
"method": "SUBSCRIBE",
"params":
[
self.pair.lower()+"#trade"
],
"id": 1
}
subscribe_message = json.dumps(subscribe_message)
await self.websocket.send(subscribe_message)
async for msg in self.websocket:
msg = json.loads(msg)
if('p' in msg):
self.quotes.loc[0] = [msg['E'],float(msg['p'])]
temp_ws = BinanceQuotesWS(client,'BTCUSDT')
temp_ws.start()
When I am testing it in Jupyter and execute a cell with temp_ws.get_quotes() manually then every single time the correct dataframe with fresh quotes is returned.
Though in my program I need to have some infinite loop and there comes up an error.
while(True):
quotes = temp_ws.get_quotes()
print(quotes)
time.sleep(3)
The quotes DF is always empty but I can't sort out why (probably because the while cycle is blocking). I will be glad if someone could help to sort out the issue (and give some hints if anything else could be improved in the code in terms of async requests). Thank you.

You could use asyncio.sleep to create async function
async def display(self):
while True:
await asyncio.sleep(3)
quotes = self.get_quotes()
print('time:', quotes['Timestamp'][0], 'price:', quotes['Price'][0])
and add it to loop
self.result2 = self.loop.create_task(self.display())
and then you can run all in the same loop
temp_ws.loop.run_forever()
If you not use run_forever() then it not run connect() - and you don't get values in your standard loop. But this loop has to runs all time and it can't runs at the same time with normal loop (which also has to run all the time). One of the loop would have to run in separated thread.
But await (whit asyncio.sleep) resolves problem. When it sleeps in while True then it goes to other functions and it can run other code - and later some other code uses await and then it can go back to while True.
Maybe in Jupyter it could work with run_forever() because they add many extra functions to make life easier (and elements used in Jupyter may need this loop to work correctly) but in normal program you have to use run_forever() manually.
Minimal working code:
import pandas as pd
import json
import websockets
import asyncio
import time
class BinanceQuotesWS:
def __init__(self,client,pair):
self.quotes = pd.DataFrame(columns=['Timestamp','Price'])
self.pair = pair
self.socket='wss://fstream.binance.com/ws'
self.websocket = None
self.loop = None
self.result = None
def get_quotes(self):
return self.quotes
def start(self):
self.loop = asyncio.get_event_loop()
self.result = self.loop.create_task(self.connect())
self.result2 = self.loop.create_task(self.display())
async def connect(self):
self.websocket = await websockets.connect(self.socket)
await self.subscribe_quotes()
async def subscribe_quotes(self):
subscribe_message = {
"method": "SUBSCRIBE",
"params": [
self.pair.lower()+"#trade"
],
"id": 1
}
subscribe_message = json.dumps(subscribe_message)
await self.websocket.send(subscribe_message)
async for msg in self.websocket:
msg = json.loads(msg)
if('p' in msg):
self.quotes.loc[0] = [msg['E'],float(msg['p'])]
#print(self.quotes)
async def display(self):
while True:
await asyncio.sleep(3)
quotes = self.get_quotes()
print('time:', quotes['Timestamp'][0], 'price:', quotes['Price'][0])
client = ''
temp_ws = BinanceQuotesWS(client,'BTCUSDT')
temp_ws.start()
temp_ws.loop.run_forever()

Processing queue with asyncio only fires after all items are placed in queue using asyncio in python

I am trying to create some code in python that will put data from a generator (currently a simple counting loop but will be a sensor data at some point) and place it in a queue. Once in a queue i want to pull data off it and send it over a TCP connection. This is a great time to use asyncio but I am doing something wrong.
Currently, the script will process all the numbers and does not return anything. Ideally I would want to make sure I have something in the queue so it never empties and send a set amount of data over say like 5 numbers everytime. How can I achieve this?
import asyncio
import random
class responder():
def __init__(self, parent=None):
super().__init__()
async def produce(self,queue, n):
for x in range(n):
# produce an item
print('producing {}/{}'.format(x, n))
# simulate i/o operation using sleep
await asyncio.sleep(random.random())
item = str(x)
# put the item in the queue
await queue.put(item)
async def consume(self,queue):
while True:
# wait for an item from the producer
item = await queue.get()
# process the item
print('consuming {}...'.format(item))
# simulate i/o operation using sleep
await asyncio.sleep(random.random())
# Notify the queue that the item has been processed
queue.task_done()
async def run(self,n):
queue = asyncio.Queue()
# schedule the consumer
self.consumer = asyncio.ensure_future(self.consume(queue))
# run the producer and wait for completion
await self.produce(queue, n)
# wait until the consumer has processed all items
await queue.join()
# the consumer is still awaiting for an item, cancel it
self.consumer.cancel()
async def handle_echo(self,reader, writer):
data = await reader.read(100)
message = data.decode()
addr = writer.get_extra_info('peername')
print("Received %r from %r" % (message, addr))
if (message == 'START_RUN'):
data = await self.run(10)
print("Send: %i" % data)
writer.write(data)
await writer.drain()
else:
print("Send: %r" % message)
writer.write(message)
await writer.drain()
print("Close the client socket")
writer.close()
def launch_server(self):
self.loop = asyncio.get_event_loop()
self.coro = asyncio.start_server(self.handle_echo, '127.0.0.1', 7780, loop=self.loop)
self.server = self.loop.run_until_complete(self.coro)
# Serve requests until Ctrl+C is pressed
print('Serving on {}'.format(self.server.sockets[0].getsockname()))
try:
self.loop.run_forever()
except KeyboardInterrupt:
pass
finally:
# Close the server
self.server.close()
self.loop.run_until_complete(self.server.wait_closed())
self.loop.close()
def main():
server = responder()
server.launch_server()
if __name__ == '__main__':
main()
The code generates the number stream but it runs through the entire list before moving on. Further I never get a value back.
My client code (which never gets anything back)
import asyncio
async def capture_stream(reader):
while not reader.at_eof:
data = await reader.read(100)
print( f'{who} received {len(data)} bytes' )
async def tcp_echo_client(message, loop):
reader, writer = await asyncio.open_connection('127.0.0.1',7780,loop=loop)
print('Send: %r' % message)
writer.write(message.encode())
if (message == "START_RUN"):
data = await reader.read(100)
print('Received: %r' % data.decode())
else:
collect_data = asyncio.create_task(capture_stream)
data = await collect_data
print('Close the socket')
writer.close()
message = 'START_RUN'
loop = asyncio.get_event_loop()
loop.run_until_complete(tcp_echo_client(message, loop))
loop.close()

Python non-blocking Asyncio

I'm trying to create a class which allows me to put data into it while a websocket connection is open, but i can't figure out how to make the main call to the class non-blocking. Can someone point me in the right direction?
Here is what I have so far (some extraneous code removed):
class Audio_Sender:
def __init__(self,IP_Address):
self.Remote_IP_Address = IP_Address
self.audio_queue = queue.Queue(10) #Max of 10 items
async def Connect(self):
uri = "ws://127.0.0.1:8765"
async with websockets.connect(uri) as websocket:
await websocket.send(json_voice_start)
while self.status == "Run":
if not self.audio_queue.empty():
audio_data = self.audio_queue.get()
await websocket.send(audio_data)
#pull any remaing data out:
while not self.audio_queue.empty():
audio_data = self.audio_queue.get()
await websocket.send(audio_data)
await websocket.send(json_voice_stop)
voice_response = await websocket.recv()
message = json.loads(voice_response)
print("\t- " + message["result"])
async def run_connect(self):
task = asyncio.create_task(self.Connect())
while not task.done():
print("Task is not Done")
await asyncio.sleep(1)
def go(self):
asyncio.run(self.run_connect())
#Create the Audio Sender
A = Audio_Sender("127.0.0.1","r")
#Put some data into it's queue
A.audio_queue.put(b"abc")
A.audio_queue.put(b"abc")
A.audio_queue.put(b"abc")
#Finished putting data in
A.status = "Done"
#Now send the data
#Ideally I would like to have the go part way through the queuing of
# data (above), but A.go() is blocking.. how to make it not blocking?
A.go()

So playing with this a bit more I realized i could use a thread for the go(), which works:
Updated code is as follows:
#Create the Audio Sender
A = Audio_Sender("127.0.0.1","r")
s = threading.Thread(target=A.go,args=())
s.start()
i=0
while i<10:
#Put some data into it's queue
A.audio_queue.put(b"abc")
i = i +1
print(i)
#Finished putting data in
A.status = "Done"