I'm trying to create a class which allows me to put data into it while a websocket connection is open, but i can't figure out how to make the main call to the class non-blocking. Can someone point me in the right direction?
Here is what I have so far (some extraneous code removed):
class Audio_Sender:
def __init__(self,IP_Address):
self.Remote_IP_Address = IP_Address
self.audio_queue = queue.Queue(10) #Max of 10 items
async def Connect(self):
uri = "ws://127.0.0.1:8765"
async with websockets.connect(uri) as websocket:
await websocket.send(json_voice_start)
while self.status == "Run":
if not self.audio_queue.empty():
audio_data = self.audio_queue.get()
await websocket.send(audio_data)
#pull any remaing data out:
while not self.audio_queue.empty():
audio_data = self.audio_queue.get()
await websocket.send(audio_data)
await websocket.send(json_voice_stop)
voice_response = await websocket.recv()
message = json.loads(voice_response)
print("\t- " + message["result"])
async def run_connect(self):
task = asyncio.create_task(self.Connect())
while not task.done():
print("Task is not Done")
await asyncio.sleep(1)
def go(self):
asyncio.run(self.run_connect())
#Create the Audio Sender
A = Audio_Sender("127.0.0.1","r")
#Put some data into it's queue
A.audio_queue.put(b"abc")
A.audio_queue.put(b"abc")
A.audio_queue.put(b"abc")
#Finished putting data in
A.status = "Done"
#Now send the data
#Ideally I would like to have the go part way through the queuing of
# data (above), but A.go() is blocking.. how to make it not blocking?
A.go()
So playing with this a bit more I realized i could use a thread for the go(), which works:
Updated code is as follows:
#Create the Audio Sender
A = Audio_Sender("127.0.0.1","r")
s = threading.Thread(target=A.go,args=())
s.start()
i=0
while i<10:
#Put some data into it's queue
A.audio_queue.put(b"abc")
i = i +1
print(i)
#Finished putting data in
A.status = "Done"
Related
I've been looking around and have seen a lot of information regarding asyncio. I'm having trouble creating a program that won't terminate as long as the background task is running.
def loop_test():
print("task is running")
time.sleep(2)
print("task is finished")
async def start_pipeline(self):
print("Starting TD Stream")
# Build data pipeline
await self.td_stream_client.build_pipeline()
data_response_count = 0
self.streaming = True
# Keep going while receiving data
while self.streaming:
print("Streaming")
data = await self.td_stream_client.start_pipeline()
# Parse if data inside
if 'data' in data:
content = data['data'][0]['content']
print("Key: {}".format(content[0]['key']))
pprint.pprint(content, indent=4)
print('-' * 80)
data_response_count += 1
print("Done with while loop")
async def main():
_ = asyncio.create_task(td_stream_client.start_pipeline())
coro = asyncio.to_thread(TDA_Streaming.loop_test)
await coro
asyncio.run(main())
The idea of the program is to have a background task that streams data from an API to my program. While this is happening, I want to be able to do other things. Maybe have manual input...maybe have a GUI where I can interact with things.
The issue is that my program terminates as soon as the master thread finishes. How do I prevent this from happening? If I have a while loop with an "input" call, this input blocks the program. What is the best way to proceed?
Please wait for the background task explicitly:
def loop_test():
print("task is running")
time.sleep(2)
print("task is finished")
async def start_pipeline(self):
print("Starting TD Stream")
# Build data pipeline
await self.td_stream_client.build_pipeline()
data_response_count = 0
self.streaming = True
# Keep going while receiving data
while self.streaming:
print("Streaming")
data = await self.td_stream_client.start_pipeline()
# Parse if data inside
if 'data' in data:
content = data['data'][0]['content']
print("Key: {}".format(content[0]['key']))
pprint.pprint(content, indent=4)
print('-' * 80)
data_response_count += 1
print("Done with while loop")
async def main():
background_task = asyncio.create_task(td_stream_client.start_pipeline())
coro = asyncio.to_thread(TDA_Streaming.loop_test)
await coro
await background_task
asyncio.run(main())
i am handling data from 2 websocket servers and i would like to know whats the fastest way to handle both connections in the same time given that the 1st connection would send data every 0.1-10ms.
what i am doing so far is:
import json
import websockets
async def run():
async with websockets.connect("ws://localhost:8546/") as ws1:
async with websockets.connect(uri="wss://api.blxrbdn.com/ws", extra_headers = {"Authorization": "apikey") as ws2:
sub1 = await ws1.send("subscription 1")
sub2 = await ws2.send("subscription 2")
while True:
try:
msg1 = await ws1.recv()
msg1 = json.loads(msg1)
msg2 = await ws2.recv()
msg2 = json.loads(msg2)
# process msg1 & msg2
except Exception as e:
print(e, flush=True)
asyncio.run(run())
As stated in the comments, try to handle each connection in its own coroutine. Here is small example:
import asyncio
import websockets
async def worker(ws, msg, t):
while True:
sub = await ws.send(msg)
print("Received from the server:", await ws.recv())
await asyncio.sleep(t)
async def run():
url1 = "ws://localhost:8765/"
url2 = "ws://something_different:8765/"
async with websockets.connect(url1) as ws1, websockets.connect(url2) as ws2:
await asyncio.gather(worker(ws1, "sub1", 1), worker(ws2, "sub2", 2))
asyncio.run(run())
Below is (working) code for a generic websocket streamer.
It creates a daemon thread from which performs asyncio.run(...).
The asyncio code spawns 2 tasks, which never complete.
How to correctly destroy this object?
One of the tasks is executing a keepalive 'ping', so I can easily exit that loop using a flag. But the other is blocking on a message from the websocket.
import json
import aiohttp
import asyncio
import gzip
import asyncio
from threading import Thread
class WebSocket:
KEEPALIVE_INTERVAL_S = 10
def __init__(self, url, on_connect, on_msg):
self.url = url
self.on_connect = on_connect
self.on_msg = on_msg
self.streams = {}
self.worker_thread = Thread(name='WebSocket', target=self.thread_func, daemon=True).start()
def thread_func(self):
asyncio.run(self.aio_run())
async def aio_run(self):
async with aiohttp.ClientSession() as session:
self.ws = await session.ws_connect(self.url)
await self.on_connect(self)
async def ping():
while True:
print('KEEPALIVE')
await self.ws.ping()
await asyncio.sleep(WebSocket.KEEPALIVE_INTERVAL_S)
async def main_loop():
async for msg in self.ws:
def extract_data(msg):
if msg.type == aiohttp.WSMsgType.BINARY:
as_bytes = gzip.decompress(msg.data)
as_string = as_bytes.decode('utf8')
as_json = json.loads(as_string)
return as_json
elif msg.type == aiohttp.WSMsgType.TEXT:
return json.loads(msg.data)
elif msg.type == aiohttp.WSMsgType.ERROR:
print('⛔️ aiohttp.WSMsgType.ERROR')
return msg.data
data = extract_data(msg)
self.on_msg(data)
# May want this approach if we want to handle graceful shutdown
# W.task_ping = asyncio.create_task(ping())
# W.task_main_loop = asyncio.create_task(main_loop())
await asyncio.gather(
ping(),
main_loop()
)
async def send_json(self, J):
await self.ws.send_json(J)
I'd suggest the use of asyncio.run_coroutine_threadsafe instead of asyncio.run. It returns a concurrent.futures.Future object which you can cancel:
def thread_func(self):
self.future = asyncio.run_coroutine_threadsafe(
self.aio_run(),
asyncio.get_event_loop()
)
# somewhere else
self.future.cancel()
Another approach would be to make ping and main_loop a task, and cancel them when necessary:
# inside `aio_run`
self.task_ping = asyncio.create_task(ping())
self.main_loop_task = asyncio.create_task(main_loop())
await asyncio.gather(
self.task_ping,
self.main_loop_task
return_exceptions=True
)
# somewhere else
self.task_ping.cancel()
self.main_loop_task.cancel()
This doesn't change the fact that aio_run should also be called with asyncio.run_coroutine_threadsafe. asyncio.run should be used as a main entry point for asyncio programs and should be only called once.
I would like to suggest one more variation of the solution. When finishing coroutines (tasks), I prefer minimizing the use of cancel() (but not excluding), since sometimes it can make it difficult to debug business logic (keep in mind that asyncio.CancelledError does not inherit from an Exception).
In your case, the code might look like this(only changes):
class WebSocket:
KEEPALIVE_INTERVAL_S = 10
def __init__(self, url, on_connect, on_msg):
# ...
self.worker_thread = Thread(name='WebSocket', target=self.thread_func)
self.worker_thread.start()
async def aio_run(self):
self._loop = asyncio.get_event_loop()
# ...
self._ping_task = asyncio.create_task(ping())
self._main_task = asyncio.create_task(main_loop())
await asyncio.gather(
self._ping_task,
self._main_task,
return_exceptions=True
)
# ...
async def stop_ping(self):
self._ping_task.cancel()
try:
await self._ping_task
except asyncio.CancelledError:
pass
async def _stop(self):
# wait ping end before socket closing
await self.stop_ping()
# lead to correct exit from `async for msg in self.ws`
await self.ws.close()
def stop(self):
# wait stopping ping and closing socket
asyncio.run_coroutine_threadsafe(
self._stop(), self._loop
).result()
self.worker_thread.join() # wait thread finish
I am trying to create some code in python that will put data from a generator (currently a simple counting loop but will be a sensor data at some point) and place it in a queue. Once in a queue i want to pull data off it and send it over a TCP connection. This is a great time to use asyncio but I am doing something wrong.
Currently, the script will process all the numbers and does not return anything. Ideally I would want to make sure I have something in the queue so it never empties and send a set amount of data over say like 5 numbers everytime. How can I achieve this?
import asyncio
import random
class responder():
def __init__(self, parent=None):
super().__init__()
async def produce(self,queue, n):
for x in range(n):
# produce an item
print('producing {}/{}'.format(x, n))
# simulate i/o operation using sleep
await asyncio.sleep(random.random())
item = str(x)
# put the item in the queue
await queue.put(item)
async def consume(self,queue):
while True:
# wait for an item from the producer
item = await queue.get()
# process the item
print('consuming {}...'.format(item))
# simulate i/o operation using sleep
await asyncio.sleep(random.random())
# Notify the queue that the item has been processed
queue.task_done()
async def run(self,n):
queue = asyncio.Queue()
# schedule the consumer
self.consumer = asyncio.ensure_future(self.consume(queue))
# run the producer and wait for completion
await self.produce(queue, n)
# wait until the consumer has processed all items
await queue.join()
# the consumer is still awaiting for an item, cancel it
self.consumer.cancel()
async def handle_echo(self,reader, writer):
data = await reader.read(100)
message = data.decode()
addr = writer.get_extra_info('peername')
print("Received %r from %r" % (message, addr))
if (message == 'START_RUN'):
data = await self.run(10)
print("Send: %i" % data)
writer.write(data)
await writer.drain()
else:
print("Send: %r" % message)
writer.write(message)
await writer.drain()
print("Close the client socket")
writer.close()
def launch_server(self):
self.loop = asyncio.get_event_loop()
self.coro = asyncio.start_server(self.handle_echo, '127.0.0.1', 7780, loop=self.loop)
self.server = self.loop.run_until_complete(self.coro)
# Serve requests until Ctrl+C is pressed
print('Serving on {}'.format(self.server.sockets[0].getsockname()))
try:
self.loop.run_forever()
except KeyboardInterrupt:
pass
finally:
# Close the server
self.server.close()
self.loop.run_until_complete(self.server.wait_closed())
self.loop.close()
def main():
server = responder()
server.launch_server()
if __name__ == '__main__':
main()
The code generates the number stream but it runs through the entire list before moving on. Further I never get a value back.
My client code (which never gets anything back)
import asyncio
async def capture_stream(reader):
while not reader.at_eof:
data = await reader.read(100)
print( f'{who} received {len(data)} bytes' )
async def tcp_echo_client(message, loop):
reader, writer = await asyncio.open_connection('127.0.0.1',7780,loop=loop)
print('Send: %r' % message)
writer.write(message.encode())
if (message == "START_RUN"):
data = await reader.read(100)
print('Received: %r' % data.decode())
else:
collect_data = asyncio.create_task(capture_stream)
data = await collect_data
print('Close the socket')
writer.close()
message = 'START_RUN'
loop = asyncio.get_event_loop()
loop.run_until_complete(tcp_echo_client(message, loop))
loop.close()
I have a code block to streaming online data, but in the mean while, I want another block to run from time to time for analysis.
class steaming_price():
def on_open(ws):
print('opened connection')
def on_close(ws):
print('closed connection')
def on_message(ws, message):
global closes, in_position, current_time
print('received message')
json_message = json.loads(message)
#pprint.pprint(json_message)
candle = json_message['k'] #all kline data
is_candle_closed = candle['x'] #if its closed
close = candle['c'] #the Close Price
#Print the Close Price
if is_candle_closed:
print("candle closed at {}".format(close))
closes.append(float(close))
current_time.append(datetime.datetime.now())
print(f"Time:{datetime.datetime.now()} Close:{close}")
ws = websocket.WebSocketApp(SOCKET, on_open=on_open, on_close=on_close, on_message=on_message)
ws.run_forever()
The following should run every 1 minutes to do analysis
class strategy():
def __init__(self):
self.closes = deque(maxlen=500)
def strategy(self, data):
self.macd, self.macd_signal, self.macd_hist = talib.MACD(data, fastperiod=12, slowperiod=26, signalperiod=9)
return self.macd, self.macd_signal, self.macd_hist
First block of code would take data each second.
I would like to have the second block of code run in parallel every 1 minute to do analysis. For example, calculate 20 bar Moving Average and issue a buy order.
I thought about async, but it only run once(I couldn't get it to work either).
Any ideas? Much appriciated
Easy solution would be to check on new message arrival how much time passed since last message was received and then if it's longer than 1 minute, do analysis on in the same function.
Another solution that you mentioned would require using asyncio websockets library:
server:
import asyncio
import json
from random import random
import websockets
async def hello(websocket, _):
while True:
value = random() * 50 + 50
value = json.dumps({"v": value})
await websocket.send(value)
print(f"sent {value}")
await asyncio.sleep(1)
start_server = websockets.serve(hello, "localhost", 8080)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()
client:
import asyncio
import json
from collections import deque
import websockets
q = deque(maxlen=5)
async def calc_sum():
while True:
data = list(q)
print(f"sum {sum(data)}")
await asyncio.sleep(3.6)
async def hello():
uri = "ws://localhost:8080"
async with websockets.connect(uri) as websocket:
while True:
value = await websocket.recv()
value = json.loads(value)
print(f"recv {value}")
q.append(value['v'])
event_loop = asyncio.get_event_loop()
event_loop.create_task(calc_sum())
event_loop.run_until_complete(hello())