Hello I am wanting to create a client socket via python, and I found this example (https://stackoverflow.com/a/49918082/12354066). The only problem I am wondering about is, I have a whole other program I want to implement this with, and it seems loop.run_until_complete(asyncio.wait(tasks)) is blocking the whole thread and not allowing me to execute any more functions i.e print(1) after loop.run_until_complete(asyncio.wait(tasks)). I want to be able to listen & send messages, but I also want to be able to execute other after I begin listening, maybe this is better suited for threads and not async (I don't know much async..)
import websockets
import asyncio
class WebSocketClient():
def __init__(self):
pass
async def connect(self):
'''
Connecting to webSocket server
websockets.client.connect returns a WebSocketClientProtocol, which is used to send and receive messages
'''
self.connection = await websockets.client.connect('ws://127.0.0.1:8765')
if self.connection.open:
print('Connection stablished. Client correcly connected')
# Send greeting
await self.sendMessage('Hey server, this is webSocket client')
return self.connection
async def sendMessage(self, message):
'''
Sending message to webSocket server
'''
await self.connection.send(message)
async def receiveMessage(self, connection):
'''
Receiving all server messages and handling them
'''
while True:
try:
message = await connection.recv()
print('Received message from server: ' + str(message))
except websockets.exceptions.ConnectionClosed:
print('Connection with server closed')
break
async def heartbeat(self, connection):
'''
Sending heartbeat to server every 5 seconds
Ping - pong messages to verify connection is alive
'''
while True:
try:
await connection.send('ping')
await asyncio.sleep(5)
except websockets.exceptions.ConnectionClosed:
print('Connection with server closed')
break
main:
import asyncio
from webSocketClient import WebSocketClient
if __name__ == '__main__':
# Creating client object
client = WebSocketClient()
loop = asyncio.get_event_loop()
# Start connection and get client connection protocol
connection = loop.run_until_complete(client.connect())
# Start listener and heartbeat
tasks = [
asyncio.ensure_future(client.heartbeat(connection)),
asyncio.ensure_future(client.receiveMessage(connection)),
]
loop.run_until_complete(asyncio.wait(tasks))
print(1) # never gets executed
Related
I've adapted this TCP echo server with streams example to my needs so that I can have multiple clients send data to my device:
import asyncio
async def handle_echo(reader, writer):
data = await reader.read(100)
message = data.decode()
addr = writer.get_extra_info('peername')
print(f"Received {message!r} from {addr!r}")
print(f"Send: {message!r}")
writer.write(data)
await writer.drain()
print("Close the connection")
writer.close()
async def main():
server = await asyncio.start_server(
handle_echo, '127.0.0.1', 8888)
addrs = ', '.join(str(sock.getsockname()) for sock in server.sockets)
print(f'Serving on {addrs}')
async with server:
await server.serve_forever()
asyncio.run(main())
The issue that I'm having is that the clients are essentially producers for another task, but I'm not sure how to be able to consume the data. I've tried declaring and passing asyncio queues into the server and into the callback function, but I've had no luck, as the handle accepts only 2 arguments and I can't declare a queue that's inside the module as it won't be a part of the event loop that's declared in the main task.
Is there a way to do this without going back to sockets?
First: the fact that no asyncio loop is running when module level code is executed does not block the creation of an asyncio.Queue instance - just do it, if you want to use asyncio.Queue s at all: but any data structure will work for you, and maybe using a collections.deque, which is synchronous, will require less boiler plate when putting/retrieving content for it: since you will already have the data to be put/consumed in the queue, it being synchronous won't make any difference.
Second: if you don't want to have a module-level data structure for that, just create a class wrapping your handler: it will then get the reference to self. The same class can wrap other methods or code that will consume your data - for example, an "awaitable get".
import time
from collections import deque
class Handler:
def __init__(self):
self.queue = deque()
async def get(self, timeout=1):
start = time.time()
while time.time() - start <= timeout:
if not self.queue:
return self.queue.popleft()
await asyncio.sleep(.0001)
async def handle_echo(self, reader, writer):
...
self.queue.append(message)
...
async def main():
handler = Handler()
server = await asyncio.start_server(
handler.handle_echo, '127.0.0.1', 8888)
addrs = ', '.join(str(sock.getsockname()) for sock in server.sockets)
print(f'Serving on {addrs}')
# pass the handler instance into other async code that will
# consume the messages:
...
async with server:
await server.serve_forever()
asyncio.run(main())
I try to create a websocket server, I wanna make a client to exchange data from server to client, but now my data from other process, I need to make a queue accept data from other process, it makes my main websocket function blocked, the final result is that could not reconnect after client connection break, I think it blocked in the code of queue.
Here is my part of my code:
class RecorderEventHook(object):
def __init__(self, high_event_mq):
self.high_event_mq = high_event_mq
self.msg = None
self.loop = None
# #wrap_keep_alive
async def on_msg_event(self, websocket):
try:
# async for message in websocket:
while True:
msg = self.high_event_mq.get()
await websocket.send(json.dumps(msg))
# msg
except Exception as error:
print(error)
async def event_controller(self):
await websockets.serve(self.on_msg_event, 'localhost', 8888)
def start(self):
loop = asyncio.new_event_loop()
loop.create_task(self.event_controller())
loop.run_forever()
I try to save connected websocket object and using in other thread(in same process), but it failed and mentions
"xxxx" function never waited
I want to be able to receive data from other processes without affecting the normal reconnection of the client.
Anybody help and big appreciate.
I have a streams server that handles multiple independent clients. When I shut it down I want to notify all clients that the server has shut down.
I figured out how to close the server to new connections, but not how to cancel the specific handlers waiting for client data.
So far the only solution I found is to cancel all tasks in the loop, but this doesn't work for me as I have other tasks that must finish their jobs first.
Does asyncio provide some interface for this or do I have to keep track of all connections myself and cancel them once the server shuts down? I would prefer if the connection handler catches an exception when it calls await reader.readuntil() and not in the middle of execution, but this is not required.
Right now the client looses connection without warning. With this it cannot tell if it was a network issue or if the server shut down.
import asyncio
import signal
server = None
shutdown = False
async def important_task():
while not shutdown:
await asyncio.sleep(10)
print("I am important")
async def handle_conn(reader,writer):
print("Got connection")
try:
while True:
text = await reader.readuntil(b'\n')
# Do stuff
writer.write( text ) # Echo example
await writer.drain()
except serverShutdownException: # How do I cause something like this?
writer.write(b"Goodbye")
await writer.drain()
finally:
writer.close()
await writer.wait_closed()
def handle_sig(num,frame):
global shutdown
print(f"Caught {num}")
server.close()
shutdown = True
async def serve():
global server
server = await asyncio.start_server(handle_conn,"127.0.0.1",8080)
try:
await server.serve_forever()
except asyncio.CancelledError:
pass
await server.wait_closed()
# wait for all handlers to be done?
def main():
signal.signal(signal.SIGINT, handle_sig)
loop = asyncio.get_event_loop()
t1 = loop.create_task(serve())
t2 = loop.create_task(important_task())
loop.run_until_complete(asyncio.gather(t1,t2))
main()
Im trying to get work Rabbit MQ queues through AioPika to send messages to Socket Io server and server would send those messages to client based on there SID, UID, etc. Documentation for using rabbitmq on socket is almost non existent. I am able to send messages to exchange with rabbit mq producer but socket io server don’t receive anything. But even if i would receive some message from rabbitmq i don’t now how to send message to the client. Because if i put sio.emit(...) into def main() sio.emit is never reached because web.run_app(app) run constantly in loop.
Im using local rabbit mq. How to make the running app and sending messages so they can run independent. Thank you for every hint and help. Really appreciated.
Socket IO code:
from aiohttp import web
import socketio
import redis
mgr = socketio.AsyncAioPikaManager(channel="Socket io test")
sio = socketio.AsyncServer(client_manager=mgr)
app = web.Application()
sio.attach(app)
async def redis_sio_uid_write(sid):
client = redis.Redis(db=0)
client.set(sid, "uid")
async def redis_sio_uid_delete(sid):
client = redis.Redis(db=0)
client.srem(sid, "uid")
#sio.event
async def connect(sid, environ):
"""Connect and print "connection established" message."""
await redis_sio_uid_write(sid)
print('Connection Established', sid)
#sio.event
async def disconnect(sid):
"""Disconnect and print "disconnected from server" message."""
print('disconnected from server', sid)
def main():
web.run_app(app)
if __name__ == '__main__':
main()
'''
Rabbit MQ Producer:
import sys
import asyncio
from aio_pika import connect, Message, DeliveryMode, ExchangeType
async def main(loop):
# Perform connection
connection = await connect(loop=loop)
# Creating a channel
channel = await connection.channel()
socketio_exchange = await channel.declare_exchange(
"Socket io test", ExchangeType.FANOUT
)
message_body = b"Hello World!"
message = Message(
message_body,
delivery_mode=DeliveryMode.PERSISTENT
)
# Sending the message
await socketio_exchange.publish(message, routing_key="")
print(" [x] Sent %r" % message)
await connection.close()
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.run_until_complete(main(loop))
ok so i solve the concurrent part with threading but still dont know ho to connect the rabbit mq onto socket io server.
socketio server:
from aiohttp import web
import socketio
import redis
import time
import random
import asyncio
from threading import Thread, Event, Lock
mgr = socketio.AsyncAioPikaManager(channel="Socket io test")
sio = socketio.AsyncServer(client_manager=mgr)
app = web.Application()
sio.attach(app)
loop = asyncio.new_event_loop()
def side_thread(loop):
asyncio.set_event_loop(loop)
loop.run_forever()
thread = Thread(target=side_thread, args=(loop,), daemon=True)
thread.start()
async def redis_sio_uid_write(sid):
client = redis.Redis(db=0)
client.sadd(sid, "".join([str(random.randint(1, 10)) for _ in range(10)]))
async def redis_sio_uid_delete(sid):
client = redis.Redis(db=0)
client.spop(sid)
#sio.event
async def connect(sid, environ):
"""Connect and print "connection established" message."""
await redis_sio_uid_write(sid)
print('Connection Established', sid)
#sio.event
async def disconnect(sid):
"""Disconnect and print "disconnected from server" message."""
await redis_sio_uid_delete(sid)
print('disconnected from server', sid)
async def _data():
while 1:
await sio.emit(event="message_rabbit_mq", data=time.time())
await asyncio.sleep(5)
def main():
future = asyncio.run_coroutine_threadsafe(_data(), loop)
future.add_done_callback(web.run_app(app))
if __name__ == '__main__':
main()
The documentation for your use case is here: Emitting from external processes.
From the documentation, the example for a Redis queue is as follows:
# connect to the redis queue as an external process
external_sio = socketio.RedisManager('redis://', write_only=True)
# emit an event
external_sio.emit('my event', data={'foo': 'bar'}, room='my room')
You would need to replace RedisManager with the aiopika manager, but other than that everything works the same. In case it isn't clear, you do not need to manage anything on the RabbitMQ side, you should use the client manager class both on the server and the external process scripts.
I am trying to create a script in python that listens to multiple sockets using websockets and asyncio, the problem is that no matter what I do it only listen to the first socket I call.
I think its the infinite loop, what are my option to solve this? using threads for each sockets?
async def start_socket(self, event):
payload = json.dumps(event)
loop = asyncio.get_event_loop()
self.tasks.append(loop.create_task(
self.subscribe(event)))
# this should not block the rest of the code
await asyncio.gather(*tasks)
def test(self):
# I want to be able to add corotines at a different time
self.start_socket(event1)
# some code
self.start_socket(event2)
this is what I did eventually, that way its not blocking the main thread and all subscriptions are working in parallel.
def subscribe(self, payload):
ws = websocket.WebSocket(sslopt={"cert_reqs": ssl.CERT_NONE})
ws.connect(url)
ws.send(payload)
while True:
result = ws.recv()
print("Received '%s'" % result)
def start_thread(self, loop):
asyncio.set_event_loop(loop)
loop.run_forever()
def start_socket(self, **kwargs):
worker_loop = asyncio.new_event_loop()
worker = Thread(target=self.start_thread, args=(worker_loop,))
worker.start()
worker_loop.call_soon_threadsafe(self.subscribe, payload)
def listen(self):
self.start_socket(payload1)
# code
self.start_socket(payload2)
# code
self.start_socket(payload3)
Your code appears incomplete, but what you've shown has two issues. One is that run_until_complete accepts a coroutine object (or other kind of future), not a coroutine function. So it should be:
# note parentheses after your_async_function()
asyncio.get_event_loop().run_until_complete(your_async_function())
the problem is that no matter what I do it only listen to the first socket I call. I think its the infinite loop, what are my option to solve this? using threads for each sockets?
The infinite loop is not the problem, asyncio is designed to support such "infinite loops". The problem is that you are trying to do everything in one coroutine, whereas you should be creating one coroutine per websocket. This is not a problem, as coroutines are very lightweight.
For example (untested):
async def subscribe_all(self, payload):
loop = asyncio.get_event_loop()
# create a task for each URL
for url in url_list:
tasks.append(loop.create_task(self.subscribe_one(url, payload)))
# run all tasks in parallel
await asyncio.gather(*tasks)
async def subsribe_one(self, url, payload):
async with websockets.connect(url) as websocket:
await websocket.send(payload)
while True:
msg = await websocket.recv()
print(msg)
One way to efficiently listen to multiple websocket connections from a websocket server is to keep a list of connected clients and essentially juggle multiple conversations in parallel.
E.g. A simple server that sends random # to each connected client every few secs:
import os
import asyncio
import websockets
import random
websocket_clients = set()
async def handle_socket_connection(websocket, path):
"""Handles the whole lifecycle of each client's websocket connection."""
websocket_clients.add(websocket)
print(f'New connection from: {websocket.remote_address} ({len(websocket_clients)} total)')
try:
# This loop will keep listening on the socket until its closed.
async for raw_message in websocket:
print(f'Got: [{raw_message}] from socket [{id(websocket)}]')
except websockets.exceptions.ConnectionClosedError as cce:
pass
finally:
print(f'Disconnected from socket [{id(websocket)}]...')
websocket_clients.remove(websocket)
async def broadcast_random_number(loop):
"""Keeps sending a random # to each connected websocket client"""
while True:
for c in websocket_clients:
num = str(random.randint(10, 99))
print(f'Sending [{num}] to socket [{id(c)}]')
await c.send(num)
await asyncio.sleep(2)
if __name__ == "__main__":
loop = asyncio.get_event_loop()
try:
socket_server = websockets.serve(handle_socket_connection, 'localhost', 6789)
print(f'Started socket server: {socket_server} ...')
loop.run_until_complete(socket_server)
loop.run_until_complete(broadcast_random_number(loop))
loop.run_forever()
finally:
loop.close()
print(f"Successfully shutdown [{loop}].")
A simple client that connects to the server and listens for the numbers:
import asyncio
import random
import websockets
async def handle_message():
uri = "ws://localhost:6789"
async with websockets.connect(uri) as websocket:
msg = 'Please send me a number...'
print(f'Sending [{msg}] to [{websocket}]')
await websocket.send(msg)
while True:
got_back = await websocket.recv()
print(f"Got: {got_back}")
asyncio.get_event_loop().run_until_complete(handle_message())
Mixing up threads and asyncio is more trouble than its worth and you still have code that will block on the most wasteful steps like network IO (which is the essential benefit of using asyncio).
You need to run each coroutine asynchronously in an event loop, call any blocking calls with await and define each method that interacts with any awaitable interactions with an async
See a working e.g.: https://github.com/adnantium/websocket_client_server