Keep connection with websocket server via async websocket client - python
I want to connect WS API of Bitmex exchange(testnet.bitmex.com) and continuously fetch data.
I am using following code, it connects sucsessfully, but dont keep the connection, closing after recieving subscription data:
import time
import json
import hmac
import hashlib
secret ='7sol85dTkfWprpa03TbRFfgbnQDxpwiYP5PLspgYx_nNDSrn'
key='YPR0oojxqAR-2dp1J76BgNhT'
websocket_url = 'wss://testnet.bitmex.com/realtime?'
async def main():
expires = str(int(round(time.time())) + 5)
path = 'GET/realtime' + expires
signature = hmac.new(secret.encode(),path.encode(),hashlib.sha256).hexdigest()
async with websockets.connect(websocket_url + 'api-expires=' + expires + '&api-signature=' + signature + '&api-key=' + key) as ws:
request = {"op": "subscribe", "args": ['trade:XBTUSD', 'instrument:XBTUSD', 'quote:XBTUSD', 'position:XBTUSD']}
await ws.send(json.dumps(request))
while True:
result = await ws.recv()
print(result)
asyncio.run(main())
Here is result, you can see responses from API, but it closes after getting of subscription list.
In synchronous execution all goes fine
{"info":"Welcome to the BitMEX Realtime API.","version":"2019-07-18T21:20:16.000Z","timestamp":"2019-07-20T22:19:51.071Z","docs":"https://testnet.bitmex.com/app/wsAPI","limit":{"remaining":39}}
{"table":"trade","action":"partial","keys":[],"types":{"timestamp":"timestamp","symbol":"symbol","side":"symbol","size":"long","price":"float","tickDirection":"symbol","trdMatchID":"guid","grossValue":"long","homeNotional":"float","foreignNotional":"float"},"foreignKeys":{"symbol":"instrument","side":"side"},"attributes":{"timestamp":"sorted","symbol":"grouped"},"filter":{"symbol":"XBTUSD"},"data":[{"timestamp":"2019-07-20T22:19:43.770Z","symbol":"XBTUSD","side":"Buy","size":3,"price":10954,"tickDirection":"PlusTick","trdMatchID":"bff64535-4c5d-2a0a-8533-47e93091daa3","grossValue":27387,"homeNotional":0.00027387,"foreignNotional":3}]}
{"table":"quote","action":"partial","keys":[],"types":{"timestamp":"timestamp","symbol":"symbol","bidSize":"long","bidPrice":"float","askPrice":"float","askSize":"long"},"foreignKeys":{"symbol":"instrument"},"attributes":{"timestamp":"sorted","symbol":"grouped"},"filter":{"symbol":"XBTUSD"},"data":[{"timestamp":"2019-07-20T22:19:43.770Z","symbol":"XBTUSD","bidSize":1863,"bidPrice":10951,"askPrice":10954,"askSize":1403}]}
{"success":true,"subscribe":"trade:XBTUSD","request":{"op":"subscribe","args":["trade:XBTUSD","instrument:XBTUSD","quote:XBTUSD","position:XBTUSD"]}}
{"success":true,"subscribe":"instrument:XBTUSD","request":{"op":"subscribe","args":["trade:XBTUSD","instrument:XBTUSD","quote:XBTUSD","position:XBTUSD"]}}
{"success":true,"subscribe":"quote:XBTUSD","request":{"op":"subscribe","args":["trade:XBTUSD","instrument:XBTUSD","quote:XBTUSD","position:XBTUSD"]}}
{"success":true,"subscribe":"position:XBTUSD","request":{"op":"subscribe","args":["trade:XBTUSD","instrument:XBTUSD","quote:XBTUSD","position:XBTUSD"]}}
{"table":"position","action":"partial","keys":["account","symbol","currency"],"types":{"account":"long","symbol":"symbol","currency":"symbol","underlying":"symbol","quoteCurrency":"symbol","commission":"float","initMarginReq":"float","maintMarginReq":"float","riskLimit":"long","leverage":"float","crossMargin":"boolean","deleveragePercentile":"float","rebalancedPnl":"long","prevRealisedPnl":"long","prevUnrealisedPnl":"long","prevClosePrice":"float","openingTimestamp":"timestamp","openingQty":"long","openingCost":"long","openingComm":"long","openOrderBuyQty":"long","openOrderBuyCost":"long","openOrderBuyPremium":"long","openOrderSellQty":"long","openOrderSellCost":"long","openOrderSellPremium":"long","execBuyQty":"long","execBuyCost":"long","execSellQty":"long","execSellCost":"long","execQty":"long","execCost":"long","execComm":"long","currentTimestamp":"timestamp","currentQty":"long","currentCost":"long","currentComm":"long","realisedCost":"long","unrealisedCost":"long","grossOpenCost":"long","grossOpenPremium":"long","grossExecCost":"long","isOpen":"boolean","markPrice":"float","markValue":"long","riskValue":"long","homeNotional":"float","foreignNotional":"float","posState":"symbol","posCost":"long","posCost2":"long","posCross":"long","posInit":"long","posComm":"long","posLoss":"long","posMargin":"long","posMaint":"long","posAllowance":"long","taxableMargin":"long","initMargin":"long","maintMargin":"long","sessionMargin":"long","targetExcessMargin":"long","varMargin":"long","realisedGrossPnl":"long","realisedTax":"long","realisedPnl":"long","unrealisedGrossPnl":"long","longBankrupt":"long","shortBankrupt":"long","taxBase":"long","indicativeTaxRate":"float","indicativeTax":"long","unrealisedTax":"long","unrealisedPnl":"long","unrealisedPnlPcnt":"float","unrealisedRoePcnt":"float","simpleQty":"float","simpleCost":"float","simpleValue":"float","simplePnl":"float","simplePnlPcnt":"float","avgCostPrice":"float","avgEntryPrice":"float","breakEvenPrice":"float","marginCallPrice":"float","liquidationPrice":"float","bankruptPrice":"float","timestamp":"timestamp","lastPrice":"float","lastValue":"long"},"foreignKeys":{"symbol":"instrument"},"attributes":{"account":"sorted","symbol":"grouped","currency":"grouped","underlying":"grouped","quoteCurrency":"grouped"},"filter":{"account":92043,"symbol":"XBTUSD"},"data":[{"account":92043,"symbol":"XBTUSD","currency":"XBt","underlying":"XBT","quoteCurrency":"USD","commission":0.00075,"initMarginReq":0.25,"maintMarginReq":0.005,"riskLimit":20000000000,"leverage":4,"crossMargin":false,"deleveragePercentile":1,"rebalancedPnl":38942,"prevRealisedPnl":255686,"prevUnrealisedPnl":0,"prevClosePrice":10978.78,"openingTimestamp":"2019-07-20T22:00:00.000Z","openingQty":-100,"openingCost":991600,"openingComm":-4854,"openOrderBuyQty":0,"openOrderBuyCost":0,"openOrderBuyPremium":0,"openOrderSellQty":0,"openOrderSellCost":0,"openOrderSellPremium":0,"execBuyQty":0,"execBuyCost":0,"execSellQty":0,"execSellCost":0,"execQty":0,"execCost":0,"execComm":0,"currentTimestamp":"2019-07-20T22:19:50.061Z","currentQty":-100,"currentCost":991600,"currentComm":-4854,"realisedCost":0,"unrealisedCost":991600,"grossOpenCost":0,"grossOpenPremium":0,"grossExecCost":0,"isOpen":true,"markPrice":10966.17,"markValue":911900,"riskValue":911900,"homeNotional":-0.009119,"foreignNotional":100,"posState":"","posCost":991600,"posCost2":991600,"posCross":44500,"posInit":247900,"posComm":963,"posLoss":0,"posMargin":293363,"posMaint":5921,"posAllowance":0,"taxableMargin":0,"initMargin":0,"maintMargin":213663,"sessionMargin":0,"targetExcessMargin":0,"varMargin":0,"realisedGrossPnl":0,"realisedTax":0,"realisedPnl":4854,"unrealisedGrossPnl":-79700,"longBankrupt":0,"shortBankrupt":0,"taxBase":0,"indicativeTaxRate":0,"indicativeTax":0,"unrealisedTax":0,"unrealisedPnl":-79700,"unrealisedPnlPcnt":-0.0804,"unrealisedRoePcnt":-0.3215,"simpleQty":null,"simpleCost":null,"simpleValue":null,"simplePnl":null,"simplePnlPcnt":null,"avgCostPrice":10084.5,"avgEntryPrice":10084.5,"breakEvenPrice":10549.5,"marginCallPrice":14200.5,"liquidationPrice":14200.5,"bankruptPrice":14302,"timestamp":"2019-07-20T22:19:50.061Z","lastPrice":10966.17,"lastValue":911900}]}
Traceback (most recent call last):
File "E:/python/TRADE2/venv/2.py", line 37, in <module>
asyncio.run(main())
File "C:\Users\Padalecki\AppData\Local\Programs\Python\Python37-32\lib\asyncio\runners.py", line 43, in run
return loop.run_until_complete(main)
File "C:\Users\Padalecki\AppData\Local\Programs\Python\Python37-32\lib\asyncio\base_events.py", line 584, in run_until_complete
return future.result()
File "E:/python/TRADE2/venv/2.py", line 27, in main
result = await ws.recv()
File "E:\python\TRADE2\venv\lib\site-packages\websockets\protocol.py", line 485, in recv
await self.ensure_open()
File "E:\python\TRADE2\venv\lib\site-packages\websockets\protocol.py", line 772, in ensure_open
raise self.connection_closed_exc()
websockets.exceptions.ConnectionClosedOK: code = 1000 (OK), no reason
I tried to use run_forever():
loop = asyncio.get_event_loop()
loop.create_task(main())
loop.run_forever()
Same result
it did not helped
Your while loop is outside of the connect() context manager block, after the block is exited the connection is closed. You just need to indent some of your code
async with websockets.connect(websocket_url + 'api-expires=' + expires + '&api-signature=' + signature + '&api-key=' + key) as ws:
request = {"op": "subscribe", "args": ['trade:XBTUSD', 'instrument:XBTUSD', 'quote:XBTUSD', 'position:XBTUSD']}
await ws.send(json.dumps(request))
>>>>while True:
>>>> result = await ws.recv()
>>>> print(result)
Related
coroutine was never awaited in async-mqtt
I have a problem with asyncio, in this program I use asyncua that is for OPC protocol and asyncio_mqtt, my program should wait for connection to broker even if any problem occurred in broker side, it should wait until get connected again but this error happened: Closing Loop Reconnect to Server in 3 ... Reconnect to Server in 2 ... Traceback (most recent call last): File "C:\Users\O.Fathi\PycharmProjects\OpcProject\final_main.py", line 310, in <module> main(server_state) File "C:\Users\O.Fathi\PycharmProjects\OpcProject\final_main.py", line 300, in main clientMqtt = asyncio.SelectorEventLoop().run_until_complete(asyncio.wait([test()])) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "C:\Users\O.Fathi\AppData\Local\Programs\Python\Python311\Lib\asyncio\base_events.py", line 650, in run_until_complete return future.result() ^^^^^^^^^^^^^^^ File "C:\Users\O.Fathi\AppData\Local\Programs\Python\Python311\Lib\asyncio\tasks.py", line 415, in wait raise TypeError("Passing coroutines is forbidden, use tasks explicitly.") TypeError: Passing coroutines is forbidden, use tasks explicitly. During handling of the above exception, another exception occurred: Traceback (most recent call last): File "C:\Users\O.Fathi\PycharmProjects\OpcProject\final_main.py", line 324, in <module> asyncio.run(main(server_state)) ^^^^^^^^^^^^^^^^^^ File "C:\Users\O.Fathi\PycharmProjects\OpcProject\final_main.py", line 300, in main clientMqtt = asyncio.SelectorEventLoop().run_until_complete(asyncio.wait([test()])) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "C:\Users\O.Fathi\AppData\Local\Programs\Python\Python311\Lib\asyncio\base_events.py", line 650, in run_until_complete return future.result() ^^^^^^^^^^^^^^^ File "C:\Users\O.Fathi\AppData\Local\Programs\Python\Python311\Lib\asyncio\tasks.py", line 415, in wait raise TypeError("Passing coroutines is forbidden, use tasks explicitly.") TypeError: Passing coroutines is forbidden, use tasks explicitly. Reconnect to Server in 1 ... sys:1: RuntimeWarning: coroutine 'test' was never awaited Process finished with exit code 1 Here is some part of program, in main function, test function called and run async-mqtt: import asyncio import json import logging import time from dataStructure import * from opcConnection import * from asyncua.ua import ObjectIds from asyncua.ua.uatypes import NodeId from asyncua import Client, Node, ua from asyncio_mqtt import Client as ClientM from asyncio_mqtt import MqttError from asyncer import asyncify async def test() -> None: structPad = set_structure('>8sh') structData = set_structure('<hffbb') firstTime = True timeSync = None dataDict = {} try: logger.info("Connecting to MQTT") async with ClientM(hostname="192.168.1.51", port=1883, client_id="OPC_client") as clientMqtt: logger.info("Connection to MQTT open") async with clientMqtt.unfiltered_messages() as messages: await clientMqtt.subscribe(mqttTopic) async for message in messages: if message.topic == "TimeSync": timeSync = message.payload print(timeSync) if firstTime: await clientMqtt.publish("ready_to_Receive_opc_topic", payload="") firstTime = False if message.topic == "send_opc_tag": print(message.payload.decode("UTF-8")) dataBase, opcServers = await create_database(message) dataDict, client = await create_dataDict(dataBase) print(dataDict) logger.info( "Message %s %s", message.topic, message.payload ) if message.topic == "Receive_OPC_Server": bMessage = message.payload.decode('UTF-8') # dataBase = json.loads(bMessage["send_opc_tag"]) client_rec = await opcConnection(bMessage) if client_rec != 0: nodesTree = await catchNodes(client_rec) await clientMqtt.publish('OPC_Server_Tree', nodesTree) print("Tree Cached") else: clientMqtt.publish('OPC_Server_Tree', "") print("Not Tree") if dataDict != {}: dataDict = await get_values(dataDict, client) value = [] percentt = [] id = [] for i in dataDict: id.append(i) if dataDict[i]["nodeList"] != []: if dataDict[i]["values"][0] is None: value.append(00.00) dataDict[i]["percent"] = 00.00 percentt.append(0.0) else: value.append(dataDict[i]["values"][0]) dataDict[i]["percent"] = percentage(dataDict[i]["VMX"], dataDict[i]["VMN"], dataDict[i]["values"]) percentt.append(dataDict[i]["percent"][0]) dataDict[i]["timeStamp"] = timeSync dataDict[i]['bufferSize'] = buffer_data_get_padding(structPad, dataDict[i]["bufferSize"], 0, timeSync, 1) values = { "id": id, "values": value, "percent": percentt, "buffer": estimate_buffer_size(len(value)) } buffer = values["buffer"] buffer_data_get_padding(structPad, buffer, 0, timeSync, len(value)) buffer_data_get(structData, buffer, values) print(values["values"]) await clientMqtt.publish("live_OPC_tags_value", payload=buffer) except MqttError as e: logger.error("Connection to MQTT closed: " + str(e)) except Exception: logger.exception("Connection to MQTT closed") await asyncio.sleep(3) clientMqtt = None server_state = True def main(server_state): mqttTopic = [("send_opc_tag", QOS_1), ("TimeSync", QOS_1), ("Receive_OPC_Server", QOS_1)] clientMqtt = asyncio.SelectorEventLoop().run_until_complete(asyncio.wait([test()])) if __name__ == "__main__": try: set_connection = True main(server_state) except KeyboardInterrupt: pass finally: print("Closing Loop") for i in range(3): time.sleep(1) print(f"Reconnect to Server in {3 - i} ...") asyncio.run(main(server_state))
Extend max connections - Python websockets 9.1 and simple-websocket-server 0.4.1
I wish I could have about 3k connections at once, but when running the server I'm having a bad crash (on about 500 connections in Win 10 x64 and on about 1019 connections on Ubuntu) with both of websockets libraries: Here is the stresstestscript.py which cause the issue (even if I run this one in different machines over the lan): import websocket,time serveraddr="ws://localhost:12345" ws = websocket.WebSocket() ws.connect(serveraddr) list = [] for b in range(3000): list.append(websocket.WebSocket()) count=1 for x in list: count+=1 x.connect(serveraddr) time.sleep(0.01) time.sleep(5000) Here is the server.py (the exact sample on websockets official documentation): #!/usr/bin/env python # WS server example that synchronizes state across clients import asyncio import json import logging import websockets logging.basicConfig() STATE = {"value": 0} USERS = set() def state_event(): return json.dumps({"type": "state", **STATE}) def users_event(): return json.dumps({"type": "users", "count": len(USERS)}) async def notify_state(): if USERS: # asyncio.wait doesn't accept an empty list message = state_event() await asyncio.wait([user.send(message) for user in USERS]) async def notify_users(): if USERS: # asyncio.wait doesn't accept an empty list message = users_event() await asyncio.wait([user.send(message) for user in USERS]) async def register(websocket): USERS.add(websocket) print("add") await notify_users() async def unregister(websocket): USERS.remove(websocket) await notify_users() async def counter(websocket, path): # register(websocket) sends user_event() to websocket await register(websocket) try: await websocket.send(state_event()) async for message in websocket: data = json.loads(message) if data["action"] == "minus": STATE["value"] -= 1 await notify_state() elif data["action"] == "plus": STATE["value"] += 1 await notify_state() else: logging.error("unsupported event: %s", data) finally: await unregister(websocket) start_server = websockets.serve(counter, "localhost", 12345) asyncio.get_event_loop().run_until_complete(start_server) asyncio.get_event_loop().run_forever() Here is the server error (on windows, when exceed the 500 connections opened at once): Traceback (most recent call last): File "websockettestserver.py", line 70, in <module> asyncio.get_event_loop().run_forever() File "C:\Python37-32\lib\asyncio\base_events.py", line 539, in run_forever self._run_once() File "C:\Python37-32\lib\asyncio\base_events.py", line 1739, in _run_once event_list = self._selector.select(timeout) File "C:\Python37-32\lib\selectors.py", line 323, in select r, w, _ = self._select(self._readers, self._writers, [], timeout) File "C:\Python37-32\lib\selectors.py", line 314, in _select r, w, x = select.select(r, w, w, timeout) ValueError: too many file descriptors in select() Like I said, I wish I could have more simultaneous connections, increasing the about 1000 connections on Linux to 3000-4000 (!?). How to fix this?
1.run more diff server.py on diff port 2.use nginx to Distribute the requests
Python websocket client close connection
websocket_client.py #!/usr/bin/env python # WS client example import asyncio import websockets import json import sys import os import time from aiortc import RTCPeerConnection, RTCSessionDescription pcs = [] peer_connections = 0 local_username = "epalxeis" websocket = None async def open_websocket(): global websocket uri = "ws://192.168.1.5:8080" async with websockets.connect(uri) as websocket: #send username await websocket.send(json.dumps({"type":"register","username":local_username})) async for message in websocket: message = json.loads(message) if(message["type"]=="create_peer"): await create_peer(message["username"],websocket) elif(message["type"]=="offer"): await receive_offer(message,websocket) elif(message["type"]=="answer"): receive_answer(message,websocket) elif(message["type"]=="unregister"): unregister(message,websocket) async def create_peer(username,websocket): global pcs global peer_connections global local_username pc = RTCPeerConnection() pc_index = peer_connections pcs.append([username,pc]) peer_connections = peer_connections+1 ''' add stream (video-audio to pc) ''' #pc.on("track") def on_track(track): ''' play track with PyQt5 ''' offer = await pcs[pc_index].createOffer() pcs[pc_index][1].setLocalDescription(offer) data = {"type":"offer","from":local_username,"to":username,"offer":offer} await websocket.send(json.dumps(data)) async def receive_offer(message,websocket): global pcs global peer_connections global local_username username = message["username"] offer = RTCSessionDescription(sdp=message["offer"]["sdp"], type=message["offer"]["type"]) pc = RTCPeerConnection() pc_index = peer_connections pcs.append([username,pc]) peer_connections = peer_connections+1 ''' add stream (video-audio to pc) ''' #pc.on("track") def on_track(track): ''' play track with PyQt5 ''' await pcs[pc_index][1].setRemoteDescription(offer) answer = await pcs[pc_index][1].createAnswer() await pcs[pc_index][1].setLocalDescription(answer) answer_json = {"sdp":answer.sdp,"type":"answer"} data = {"type":"answer","from":local_username,"to":username,"answer":answer_json} await websocket.send(json.dumps(data)) async def receive_answer(message,websocket): global pcs username = message["username"] answer = message["answer"] for pc in pcs: if(pc[0]==username): pc[1].setRemoteDescription(answer) async def unregister(message,websocket): global pcs global peer_connections username = message["username"] index = 0 counter = 0 for pc in pcs: if(pc[0]==username): pc[1].close() ''' PyQt5 manage for closing Output Device ''' index = counter break counter = counter + 1 del pcs[index] peer_connections = peer_connections-1 asyncio.get_event_loop().run_until_complete(open_websocket()) websocket_server.py import asyncio import websockets import json import ssl peers = () async def on_open(websocket,path): async for message in websocket: message = json.loads(message) if(message["type"]=="register"): await register(websocket,message["username"]) elif(message["type"]=="offer"): await send_offer(websocket,message) elif(message["type"]=="answer"): await send_answer(websocket,message) elif(message["type"]=="candidate"): await send_candidate(websocket,message) await unregister(websocket) async def register(websocket,username): global peers print(username+" logged in.") peers = peers + ((websocket,username),) for peer in peers: if peer[0] is not websocket: await websocket.send(json.dumps({"type": "create_peer","username":peer[1]})) async def send_offer(websocket,message): global peers offer_creator = message["from"] offer_receiver = message["to"] offer = message["offer"] print(offer_creator+" creates and sends offer to "+offer_receiver) for peer in peers: if(peer[1]==offer_receiver): await peer[0].send(json.dumps({"type": "offer","username":offer_creator,"offer":offer})) async def send_answer(websocket,message): global peers answer_creator = message["from"] answer_receiver = message["to"] answer = message["answer"] print(answer_creator+" creates and sends answer to "+answer_receiver) for peer in peers: if(peer[1]==answer_receiver): await peer[0].send(json.dumps({"type": "answer","username":answer_creator,"answer":answer})) async def send_candidate(websocket,message): global peers candidate_creator = message["from"] candidate_receiver = message["to"] candidate = message["candidate"] print(candidate_creator+" send candidate packet to "+candidate_receiver) for peer in peers: if(peer[1]==candidate_receiver): await peer[0].send(json.dumps({"type": "candidate","username":candidate_creator,"candidate":candidate})) async def unregister(websocket): global peers for peer_1 in peers: if(peer_1[0]==websocket): username = peer_1[1] print(username+" logged out.") for peer_2 in peers: if(peer_2[0] is not websocket): await peer_2[0].send(json.dumps({"type": "unregister","username":username})) peers_list = list(peers) peers_list.remove((websocket,username)) peers = tuple(peers_list) ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) ssl_context.load_cert_chain(r'C:\xampp\crt\localhost\server.crt',r'C:\xampp\crt\localhost\server.key') ssl_context = None start_server = websockets.serve(on_open, "192.168.1.5", 8080, ssl=ssl_context) asyncio.get_event_loop().run_until_complete(start_server) asyncio.get_event_loop().run_forever() First i run the websocket_server.py (this code is handling the websocket requests) Second i run the websocket_client.py After that i saw in the first cmd: epalxeis logged in. So the connection is correct. But if i close the second cmd (with the x button or with the ctr-c), i saw this error in the server's cmd: Error in connection handler Traceback (most recent call last): File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\asyncio\windows_events.py", line 453, in finish_recv return ov.getresult() OSError: [WinError 64] Το καθορισμένο όνομα δικτύου δεν είναι πια διαθέσιμο During handling of the above exception, another exception occurred: Traceback (most recent call last): File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\site-packages\websockets\protocol.py", line 827, in transfer_data message = await self.read_message() File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\site-packages\websockets\protocol.py", line 895, in read_message frame = await self.read_data_frame(max_size=self.max_size) File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\site-packages\websockets\protocol.py", line 971, in read_data_frame frame = await self.read_frame(max_size) File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\site-packages\websockets\protocol.py", line 1047, in read_frame frame = await Frame.read( File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\site-packages\websockets\framing.py", line 105, in read data = await reader(2) File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\asyncio\streams.py", line 723, in readexactly await self._wait_for_data('readexactly') File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\asyncio\streams.py", line 517, in _wait_for_data await self._waiter File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\asyncio\proactor_events.py", line 280, in _loop_reading data = fut.result() File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\asyncio\windows_events.py", line 808, in _poll value = callback(transferred, key, ov) File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\asyncio\windows_events.py", line 457, in finish_recv raise ConnectionResetError(*exc.args) ConnectionResetError: [WinError 64] Το καθορισμένο όνομα δικτύου δεν είναι πια διαθέσιμο The above exception was the direct cause of the following exception: Traceback (most recent call last): File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\site-packages\websockets\server.py", line 191, in handler await self.ws_handler(self, path) File "websocket_server.py", line 9, in on_open async for message in websocket: File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\site-packages\websockets\protocol.py", line 439, in __aiter__ yield await self.recv() File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\site-packages\websockets\protocol.py", line 509, in recv await self.ensure_open() File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\site-packages\websockets\protocol.py", line 803, in ensure_open raise self.connection_closed_exc() websockets.exceptions.ConnectionClosedError: code = 1006 (connection closed abnormally [internal]), no reason The proccess don't stop. How can i resolve the issue? Thanks in advance, Chris Pappas
This post helps you. The reason program couldn't stop is that sub threads continue to run even if main thread catches an exception and stop. Note that asyncio is implemented by thread (I pursued codes). There are two kinds of thread in Python, normal thread and daemon thread. The former continues to run if parent thread dies, the latter stops. asyncio is implemented with the former, therefore such this problem happens.
discord.py Invite Tracker
When i use a script i found on here, which allowed you to track a users invites another member to discord. The script works great, for a short while then i get the following error Task exception was never retrieved future: exception=AttributeError("'NoneType' object has no attribute 'invites'")> Traceback (most recent call last): File "H:/TheVoidv2/bot.py", line 1509, in fetch invs = await gld.invites() and this error once the bot has ran for a good few hours Task exception was never retrieved future: <Task finished name='Task-1' coro=<fetch() done, defined at C:\Users\Administrator\Desktop\TheVoidv2\bot.py:1416> exception=HTTPException('503 Service Unavailable (error code: 0): upstream connect error or disconnect/reset before headers. reset reason: connection failure')> Traceback (most recent call last): File "C:\Users\Administrator\Desktop\TheVoidv2\bot.py", line 1423, in fetch invs = await gld.invites() File "C:\Users\Administrator\AppData\Local\Programs\Python\Python38\lib\site-packages\discord\guild.py", line 1407, in invites data = await self._state.http.invites_from(self.id) File "C:\Users\Administrator\AppData\Local\Programs\Python\Python38\lib\site-packages\discord\http.py", line 225, in request raise HTTPException(r, data) discord.errors.HTTPException: 503 Service Unavailable (error code: 0): upstream connect error or disconnect/reset before headers. reset reason: connection failure import datetime import json import os import commands client = discord.Client() cfg = open("config.json", "r") tmpconfig = cfg.read() cfg.close() config = json.loads(tmpconfig) token = config["token"] guild_id = config["server-id"] logs_channel = config["logs-channel-id"] invites = {} last = "" async def fetch(): global last global invites await client.wait_until_ready() gld = client.get_guild(int(guild_id)) logs = client.get_channel(int(logs_channel)) while True: invs = await gld.invites() tmp = [] for i in invs: for s in invites: if s[0] == i.code: if int(i.uses) > s[1]: usr = gld.get_member(int(last)) testh = f"{usr.name} **joined**; Invited by **{i.inviter.name}** (**{str(i.uses)}** invites)" await logs.send(testh) tmp.append(tuple((i.code, i.uses))) invites = tmp await asyncio.sleep(4) #client.event async def on_ready(): print("ready!") await client.change_presence(activity = discord.Activity(name = "joins", type = 2)) #client.event async def on_member_join(meme): global last last = str(meme.id) client.loop.create_task(fetch()) client.run(token)
How to prevent raise asyncio.TimeoutError and continue the loop
I'm using aiohttp with limited_as_completed method to speed up scrapping (around 100 million static website pages). However, the code stops after several minutes, and returns the TimeoutError. I tried several things, but still could not prevent the raise asyncio.TimeoutError. May I ask how can I ignore the error, and continue? The code I'm running is: N=123 import html from lxml import etree import requests import asyncio import aiohttp from aiohttp import ClientSession, TCPConnector import pandas as pd import re import csv import time from itertools import islice import sys from contextlib import suppress start = time.time() data = {} data['name'] = [] filename = "C:\\Users\\xxxx"+ str(N) + ".csv" def limited_as_completed(coros, limit): futures = [ asyncio.ensure_future(c) for c in islice(coros, 0, limit) ] async def first_to_finish(): while True: await asyncio.sleep(0) for f in futures: if f.done(): futures.remove(f) try: newf = next(coros) futures.append( asyncio.ensure_future(newf)) except StopIteration as e: pass return f.result() while len(futures) > 0: yield first_to_finish() async def get_info_byid(i, url, session): async with session.get(url,timeout=20) as resp: print(url) with suppress(asyncio.TimeoutError): r = await resp.text() name = etree.HTML(r).xpath('//h2[starts-with(text(),"Customer Name")]/text()') data['name'].append(name) dataframe = pd.DataFrame(data) dataframe.to_csv(filename, index=False, sep='|') limit = 1000 async def print_when_done(tasks): for res in limited_as_completed(tasks, limit): await res url = "http://xxx.{}.html" loop = asyncio.get_event_loop() async def main(): connector = TCPConnector(limit=10) async with ClientSession(connector=connector,headers=headers,raise_for_status=False) as session: coros = (get_info_byid(i, url.format(i), session) for i in range(N,N+1000000)) await print_when_done(coros) loop.run_until_complete(main()) loop.close() print("took", time.time() - start, "seconds.") The error log is: Traceback (most recent call last): File "C:\Users\xxx.py", line 111, in <module> loop.run_until_complete(main()) File "C:\Users\xx\AppData\Local\Programs\Python\Python37-32\lib\asyncio\base_events.py", line 573, in run_until_complete return future.result() File "C:\Users\xxx.py", line 109, in main await print_when_done(coros) File "C:\Users\xxx.py", line 98, in print_when_done await res File "C:\Users\xxx.py", line 60, in first_to_finish return f.result() File "C:\Users\xxx.py", line 65, in get_info_byid async with session.get(url,timeout=20) as resp: File "C:\Users\xx\AppData\Local\Programs\Python\Python37-32\lib\site-packages\aiohttp\client.py", line 855, in __aenter__ self._resp = await self._coro File "C:\Users\xx\AppData\Local\Programs\Python\Python37-32\lib\site-packages\aiohttp\client.py", line 391, in _request await resp.start(conn) File "C:\Users\xx\AppData\Local\Programs\Python\Python37-32\lib\site-packages\aiohttp\client_reqrep.py", line 770, in start self._continue = None File "C:\Users\xx\AppData\Local\Programs\Python\Python37-32\lib\site-packages\aiohttp\helpers.py", line 673, in __exit__ raise asyncio.TimeoutError from None concurrent.futures._base.TimeoutError I have tried 1) add expect asyncio.TimeoutError: pass. Not working async def get_info_byid(i, url, session): async with session.get(url,timeout=20) as resp: print(url) try: r = await resp.text() name = etree.HTML(r).xpath('//h2[starts-with(text(),"Customer Name")]/text()') data['name'].append(name) dataframe = pd.DataFrame(data) dataframe.to_csv(filename, index=False, sep='|') except asyncio.TimeoutError: pass 2) suppress(asyncio.TimeoutError)as shown above. Not working I just learned aiohttp yesterday, so maybe there is other things wrong in my code that causes timeout error only after a few minutes' running? Thank you very much if anyone knows how to deal with it!
what #Yurii Kramarenko has done will raise Unclosed client session excecption for sure, since the session has never be properly closed. What I recommend is sth like this: import asyncio import aiohttp async def main(urls): async with aiohttp.ClientSession(timeout=self.timeout) as session: tasks=[self.do_something(session,url) for url in urls] await asyncio.gather(*tasks)
I like #jbxiaoyu answer, but the timeout kwarg seems to take a special object, so I thought I'd add you need to create a ClientTimeout object, then pass it to the Session, like this: from aiohttp import ClientSession, ClientTimeout timeout = ClientTimeout(total=600) async with ClientSession(timeout=timeout) as session: tasks=[self.do_something(session,url) for url in urls] await asyncio.gather(*tasks)
Simple example (not very good, but works fine): import asyncio from aiohttp.client import ClientSession class Wrapper: def __init__(self, session): self._session = session async def get(self, url): try: async with self._session.get(url, timeout=20) as resp: return await resp.text() except Exception as e: print(e) loop = asyncio.get_event_loop() wrapper = Wrapper(ClientSession()) responses = loop.run_until_complete( asyncio.gather( wrapper.get('http://google.com'), wrapper.get('http://google.com'), wrapper.get('http://google.com'), wrapper.get('http://google.com'), wrapper.get('http://google.com') ) ) print(responses)