Keep connection with websocket server via async websocket client - python

I want to connect WS API of Bitmex exchange(testnet.bitmex.com) and continuously fetch data.
I am using following code, it connects sucsessfully, but dont keep the connection, closing after recieving subscription data:
import time
import json
import hmac
import hashlib
secret ='7sol85dTkfWprpa03TbRFfgbnQDxpwiYP5PLspgYx_nNDSrn'
key='YPR0oojxqAR-2dp1J76BgNhT'
websocket_url = 'wss://testnet.bitmex.com/realtime?'
async def main():
expires = str(int(round(time.time())) + 5)
path = 'GET/realtime' + expires
signature = hmac.new(secret.encode(),path.encode(),hashlib.sha256).hexdigest()
async with websockets.connect(websocket_url + 'api-expires=' + expires + '&api-signature=' + signature + '&api-key=' + key) as ws:
request = {"op": "subscribe", "args": ['trade:XBTUSD', 'instrument:XBTUSD', 'quote:XBTUSD', 'position:XBTUSD']}
await ws.send(json.dumps(request))
while True:
result = await ws.recv()
print(result)
asyncio.run(main())
Here is result, you can see responses from API, but it closes after getting of subscription list.
In synchronous execution all goes fine
{"info":"Welcome to the BitMEX Realtime API.","version":"2019-07-18T21:20:16.000Z","timestamp":"2019-07-20T22:19:51.071Z","docs":"https://testnet.bitmex.com/app/wsAPI","limit":{"remaining":39}}
{"table":"trade","action":"partial","keys":[],"types":{"timestamp":"timestamp","symbol":"symbol","side":"symbol","size":"long","price":"float","tickDirection":"symbol","trdMatchID":"guid","grossValue":"long","homeNotional":"float","foreignNotional":"float"},"foreignKeys":{"symbol":"instrument","side":"side"},"attributes":{"timestamp":"sorted","symbol":"grouped"},"filter":{"symbol":"XBTUSD"},"data":[{"timestamp":"2019-07-20T22:19:43.770Z","symbol":"XBTUSD","side":"Buy","size":3,"price":10954,"tickDirection":"PlusTick","trdMatchID":"bff64535-4c5d-2a0a-8533-47e93091daa3","grossValue":27387,"homeNotional":0.00027387,"foreignNotional":3}]}
{"table":"quote","action":"partial","keys":[],"types":{"timestamp":"timestamp","symbol":"symbol","bidSize":"long","bidPrice":"float","askPrice":"float","askSize":"long"},"foreignKeys":{"symbol":"instrument"},"attributes":{"timestamp":"sorted","symbol":"grouped"},"filter":{"symbol":"XBTUSD"},"data":[{"timestamp":"2019-07-20T22:19:43.770Z","symbol":"XBTUSD","bidSize":1863,"bidPrice":10951,"askPrice":10954,"askSize":1403}]}
{"success":true,"subscribe":"trade:XBTUSD","request":{"op":"subscribe","args":["trade:XBTUSD","instrument:XBTUSD","quote:XBTUSD","position:XBTUSD"]}}
{"success":true,"subscribe":"instrument:XBTUSD","request":{"op":"subscribe","args":["trade:XBTUSD","instrument:XBTUSD","quote:XBTUSD","position:XBTUSD"]}}
{"success":true,"subscribe":"quote:XBTUSD","request":{"op":"subscribe","args":["trade:XBTUSD","instrument:XBTUSD","quote:XBTUSD","position:XBTUSD"]}}
{"success":true,"subscribe":"position:XBTUSD","request":{"op":"subscribe","args":["trade:XBTUSD","instrument:XBTUSD","quote:XBTUSD","position:XBTUSD"]}}
{"table":"position","action":"partial","keys":["account","symbol","currency"],"types":{"account":"long","symbol":"symbol","currency":"symbol","underlying":"symbol","quoteCurrency":"symbol","commission":"float","initMarginReq":"float","maintMarginReq":"float","riskLimit":"long","leverage":"float","crossMargin":"boolean","deleveragePercentile":"float","rebalancedPnl":"long","prevRealisedPnl":"long","prevUnrealisedPnl":"long","prevClosePrice":"float","openingTimestamp":"timestamp","openingQty":"long","openingCost":"long","openingComm":"long","openOrderBuyQty":"long","openOrderBuyCost":"long","openOrderBuyPremium":"long","openOrderSellQty":"long","openOrderSellCost":"long","openOrderSellPremium":"long","execBuyQty":"long","execBuyCost":"long","execSellQty":"long","execSellCost":"long","execQty":"long","execCost":"long","execComm":"long","currentTimestamp":"timestamp","currentQty":"long","currentCost":"long","currentComm":"long","realisedCost":"long","unrealisedCost":"long","grossOpenCost":"long","grossOpenPremium":"long","grossExecCost":"long","isOpen":"boolean","markPrice":"float","markValue":"long","riskValue":"long","homeNotional":"float","foreignNotional":"float","posState":"symbol","posCost":"long","posCost2":"long","posCross":"long","posInit":"long","posComm":"long","posLoss":"long","posMargin":"long","posMaint":"long","posAllowance":"long","taxableMargin":"long","initMargin":"long","maintMargin":"long","sessionMargin":"long","targetExcessMargin":"long","varMargin":"long","realisedGrossPnl":"long","realisedTax":"long","realisedPnl":"long","unrealisedGrossPnl":"long","longBankrupt":"long","shortBankrupt":"long","taxBase":"long","indicativeTaxRate":"float","indicativeTax":"long","unrealisedTax":"long","unrealisedPnl":"long","unrealisedPnlPcnt":"float","unrealisedRoePcnt":"float","simpleQty":"float","simpleCost":"float","simpleValue":"float","simplePnl":"float","simplePnlPcnt":"float","avgCostPrice":"float","avgEntryPrice":"float","breakEvenPrice":"float","marginCallPrice":"float","liquidationPrice":"float","bankruptPrice":"float","timestamp":"timestamp","lastPrice":"float","lastValue":"long"},"foreignKeys":{"symbol":"instrument"},"attributes":{"account":"sorted","symbol":"grouped","currency":"grouped","underlying":"grouped","quoteCurrency":"grouped"},"filter":{"account":92043,"symbol":"XBTUSD"},"data":[{"account":92043,"symbol":"XBTUSD","currency":"XBt","underlying":"XBT","quoteCurrency":"USD","commission":0.00075,"initMarginReq":0.25,"maintMarginReq":0.005,"riskLimit":20000000000,"leverage":4,"crossMargin":false,"deleveragePercentile":1,"rebalancedPnl":38942,"prevRealisedPnl":255686,"prevUnrealisedPnl":0,"prevClosePrice":10978.78,"openingTimestamp":"2019-07-20T22:00:00.000Z","openingQty":-100,"openingCost":991600,"openingComm":-4854,"openOrderBuyQty":0,"openOrderBuyCost":0,"openOrderBuyPremium":0,"openOrderSellQty":0,"openOrderSellCost":0,"openOrderSellPremium":0,"execBuyQty":0,"execBuyCost":0,"execSellQty":0,"execSellCost":0,"execQty":0,"execCost":0,"execComm":0,"currentTimestamp":"2019-07-20T22:19:50.061Z","currentQty":-100,"currentCost":991600,"currentComm":-4854,"realisedCost":0,"unrealisedCost":991600,"grossOpenCost":0,"grossOpenPremium":0,"grossExecCost":0,"isOpen":true,"markPrice":10966.17,"markValue":911900,"riskValue":911900,"homeNotional":-0.009119,"foreignNotional":100,"posState":"","posCost":991600,"posCost2":991600,"posCross":44500,"posInit":247900,"posComm":963,"posLoss":0,"posMargin":293363,"posMaint":5921,"posAllowance":0,"taxableMargin":0,"initMargin":0,"maintMargin":213663,"sessionMargin":0,"targetExcessMargin":0,"varMargin":0,"realisedGrossPnl":0,"realisedTax":0,"realisedPnl":4854,"unrealisedGrossPnl":-79700,"longBankrupt":0,"shortBankrupt":0,"taxBase":0,"indicativeTaxRate":0,"indicativeTax":0,"unrealisedTax":0,"unrealisedPnl":-79700,"unrealisedPnlPcnt":-0.0804,"unrealisedRoePcnt":-0.3215,"simpleQty":null,"simpleCost":null,"simpleValue":null,"simplePnl":null,"simplePnlPcnt":null,"avgCostPrice":10084.5,"avgEntryPrice":10084.5,"breakEvenPrice":10549.5,"marginCallPrice":14200.5,"liquidationPrice":14200.5,"bankruptPrice":14302,"timestamp":"2019-07-20T22:19:50.061Z","lastPrice":10966.17,"lastValue":911900}]}
Traceback (most recent call last):
File "E:/python/TRADE2/venv/2.py", line 37, in <module>
asyncio.run(main())
File "C:\Users\Padalecki\AppData\Local\Programs\Python\Python37-32\lib\asyncio\runners.py", line 43, in run
return loop.run_until_complete(main)
File "C:\Users\Padalecki\AppData\Local\Programs\Python\Python37-32\lib\asyncio\base_events.py", line 584, in run_until_complete
return future.result()
File "E:/python/TRADE2/venv/2.py", line 27, in main
result = await ws.recv()
File "E:\python\TRADE2\venv\lib\site-packages\websockets\protocol.py", line 485, in recv
await self.ensure_open()
File "E:\python\TRADE2\venv\lib\site-packages\websockets\protocol.py", line 772, in ensure_open
raise self.connection_closed_exc()
websockets.exceptions.ConnectionClosedOK: code = 1000 (OK), no reason
I tried to use run_forever():
loop = asyncio.get_event_loop()
loop.create_task(main())
loop.run_forever()
Same result
it did not helped

Your while loop is outside of the connect() context manager block, after the block is exited the connection is closed. You just need to indent some of your code
async with websockets.connect(websocket_url + 'api-expires=' + expires + '&api-signature=' + signature + '&api-key=' + key) as ws:
request = {"op": "subscribe", "args": ['trade:XBTUSD', 'instrument:XBTUSD', 'quote:XBTUSD', 'position:XBTUSD']}
await ws.send(json.dumps(request))
>>>>while True:
>>>> result = await ws.recv()
>>>> print(result)

Related

coroutine was never awaited in async-mqtt

I have a problem with asyncio, in this program I use asyncua that is for OPC protocol and asyncio_mqtt, my program should wait for connection to broker even if any problem occurred in broker side, it should wait until get connected again but this error happened:
Closing Loop
Reconnect to Server in 3 ...
Reconnect to Server in 2 ...
Traceback (most recent call last):
File "C:\Users\O.Fathi\PycharmProjects\OpcProject\final_main.py", line 310, in <module>
main(server_state)
File "C:\Users\O.Fathi\PycharmProjects\OpcProject\final_main.py", line 300, in main
clientMqtt = asyncio.SelectorEventLoop().run_until_complete(asyncio.wait([test()]))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\O.Fathi\AppData\Local\Programs\Python\Python311\Lib\asyncio\base_events.py", line 650, in run_until_complete
return future.result()
^^^^^^^^^^^^^^^
File "C:\Users\O.Fathi\AppData\Local\Programs\Python\Python311\Lib\asyncio\tasks.py", line 415, in wait
raise TypeError("Passing coroutines is forbidden, use tasks explicitly.")
TypeError: Passing coroutines is forbidden, use tasks explicitly.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\O.Fathi\PycharmProjects\OpcProject\final_main.py", line 324, in <module>
asyncio.run(main(server_state))
^^^^^^^^^^^^^^^^^^
File "C:\Users\O.Fathi\PycharmProjects\OpcProject\final_main.py", line 300, in main
clientMqtt = asyncio.SelectorEventLoop().run_until_complete(asyncio.wait([test()]))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\O.Fathi\AppData\Local\Programs\Python\Python311\Lib\asyncio\base_events.py", line 650, in run_until_complete
return future.result()
^^^^^^^^^^^^^^^
File "C:\Users\O.Fathi\AppData\Local\Programs\Python\Python311\Lib\asyncio\tasks.py", line 415, in wait
raise TypeError("Passing coroutines is forbidden, use tasks explicitly.")
TypeError: Passing coroutines is forbidden, use tasks explicitly.
Reconnect to Server in 1 ...
sys:1: RuntimeWarning: coroutine 'test' was never awaited
Process finished with exit code 1
Here is some part of program, in main function, test function called and run async-mqtt:
import asyncio
import json
import logging
import time
from dataStructure import *
from opcConnection import *
from asyncua.ua import ObjectIds
from asyncua.ua.uatypes import NodeId
from asyncua import Client, Node, ua
from asyncio_mqtt import Client as ClientM
from asyncio_mqtt import MqttError
from asyncer import asyncify
async def test() -> None:
structPad = set_structure('>8sh')
structData = set_structure('<hffbb')
firstTime = True
timeSync = None
dataDict = {}
try:
logger.info("Connecting to MQTT")
async with ClientM(hostname="192.168.1.51", port=1883, client_id="OPC_client") as clientMqtt:
logger.info("Connection to MQTT open")
async with clientMqtt.unfiltered_messages() as messages:
await clientMqtt.subscribe(mqttTopic)
async for message in messages:
if message.topic == "TimeSync":
timeSync = message.payload
print(timeSync)
if firstTime:
await clientMqtt.publish("ready_to_Receive_opc_topic", payload="")
firstTime = False
if message.topic == "send_opc_tag":
print(message.payload.decode("UTF-8"))
dataBase, opcServers = await create_database(message)
dataDict, client = await create_dataDict(dataBase)
print(dataDict)
logger.info(
"Message %s %s", message.topic, message.payload
)
if message.topic == "Receive_OPC_Server":
bMessage = message.payload.decode('UTF-8')
# dataBase = json.loads(bMessage["send_opc_tag"])
client_rec = await opcConnection(bMessage)
if client_rec != 0:
nodesTree = await catchNodes(client_rec)
await clientMqtt.publish('OPC_Server_Tree', nodesTree)
print("Tree Cached")
else:
clientMqtt.publish('OPC_Server_Tree', "")
print("Not Tree")
if dataDict != {}:
dataDict = await get_values(dataDict, client)
value = []
percentt = []
id = []
for i in dataDict:
id.append(i)
if dataDict[i]["nodeList"] != []:
if dataDict[i]["values"][0] is None:
value.append(00.00)
dataDict[i]["percent"] = 00.00
percentt.append(0.0)
else:
value.append(dataDict[i]["values"][0])
dataDict[i]["percent"] = percentage(dataDict[i]["VMX"], dataDict[i]["VMN"],
dataDict[i]["values"])
percentt.append(dataDict[i]["percent"][0])
dataDict[i]["timeStamp"] = timeSync
dataDict[i]['bufferSize'] = buffer_data_get_padding(structPad, dataDict[i]["bufferSize"], 0,
timeSync, 1)
values = {
"id": id,
"values": value,
"percent": percentt,
"buffer": estimate_buffer_size(len(value))
}
buffer = values["buffer"]
buffer_data_get_padding(structPad, buffer, 0, timeSync, len(value))
buffer_data_get(structData, buffer, values)
print(values["values"])
await clientMqtt.publish("live_OPC_tags_value", payload=buffer)
except MqttError as e:
logger.error("Connection to MQTT closed: " + str(e))
except Exception:
logger.exception("Connection to MQTT closed")
await asyncio.sleep(3)
clientMqtt = None
server_state = True
def main(server_state):
mqttTopic = [("send_opc_tag", QOS_1), ("TimeSync", QOS_1), ("Receive_OPC_Server", QOS_1)]
clientMqtt = asyncio.SelectorEventLoop().run_until_complete(asyncio.wait([test()]))
if __name__ == "__main__":
try:
set_connection = True
main(server_state)
except KeyboardInterrupt:
pass
finally:
print("Closing Loop")
for i in range(3):
time.sleep(1)
print(f"Reconnect to Server in {3 - i} ...")
asyncio.run(main(server_state))

Extend max connections - Python websockets 9.1 and simple-websocket-server 0.4.1

I wish I could have about 3k connections at once, but when running the server I'm having a bad crash (on about 500 connections in Win 10 x64 and on about 1019 connections on Ubuntu) with both of websockets libraries:
Here is the stresstestscript.py which cause the issue (even if I run this one in different machines over the lan):
import websocket,time
serveraddr="ws://localhost:12345"
ws = websocket.WebSocket()
ws.connect(serveraddr)
list = []
for b in range(3000):
list.append(websocket.WebSocket())
count=1
for x in list:
count+=1
x.connect(serveraddr)
time.sleep(0.01)
time.sleep(5000)
Here is the server.py (the exact sample on websockets official documentation):
#!/usr/bin/env python
# WS server example that synchronizes state across clients
import asyncio
import json
import logging
import websockets
logging.basicConfig()
STATE = {"value": 0}
USERS = set()
def state_event():
return json.dumps({"type": "state", **STATE})
def users_event():
return json.dumps({"type": "users", "count": len(USERS)})
async def notify_state():
if USERS: # asyncio.wait doesn't accept an empty list
message = state_event()
await asyncio.wait([user.send(message) for user in USERS])
async def notify_users():
if USERS: # asyncio.wait doesn't accept an empty list
message = users_event()
await asyncio.wait([user.send(message) for user in USERS])
async def register(websocket):
USERS.add(websocket)
print("add")
await notify_users()
async def unregister(websocket):
USERS.remove(websocket)
await notify_users()
async def counter(websocket, path):
# register(websocket) sends user_event() to websocket
await register(websocket)
try:
await websocket.send(state_event())
async for message in websocket:
data = json.loads(message)
if data["action"] == "minus":
STATE["value"] -= 1
await notify_state()
elif data["action"] == "plus":
STATE["value"] += 1
await notify_state()
else:
logging.error("unsupported event: %s", data)
finally:
await unregister(websocket)
start_server = websockets.serve(counter, "localhost", 12345)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()
Here is the server error (on windows, when exceed the 500 connections opened at once):
Traceback (most recent call last):
File "websockettestserver.py", line 70, in <module>
asyncio.get_event_loop().run_forever()
File "C:\Python37-32\lib\asyncio\base_events.py", line 539, in run_forever
self._run_once()
File "C:\Python37-32\lib\asyncio\base_events.py", line 1739, in _run_once
event_list = self._selector.select(timeout)
File "C:\Python37-32\lib\selectors.py", line 323, in select
r, w, _ = self._select(self._readers, self._writers, [], timeout)
File "C:\Python37-32\lib\selectors.py", line 314, in _select
r, w, x = select.select(r, w, w, timeout)
ValueError: too many file descriptors in select()
Like I said, I wish I could have more simultaneous connections, increasing the about 1000 connections on Linux to 3000-4000 (!?).
How to fix this?
1.run more diff server.py on diff port
2.use nginx to Distribute the requests

Python websocket client close connection

websocket_client.py
#!/usr/bin/env python
# WS client example
import asyncio
import websockets
import json
import sys
import os
import time
from aiortc import RTCPeerConnection, RTCSessionDescription
pcs = []
peer_connections = 0
local_username = "epalxeis"
websocket = None
async def open_websocket():
global websocket
uri = "ws://192.168.1.5:8080"
async with websockets.connect(uri) as websocket:
#send username
await websocket.send(json.dumps({"type":"register","username":local_username}))
async for message in websocket:
message = json.loads(message)
if(message["type"]=="create_peer"):
await create_peer(message["username"],websocket)
elif(message["type"]=="offer"):
await receive_offer(message,websocket)
elif(message["type"]=="answer"):
receive_answer(message,websocket)
elif(message["type"]=="unregister"):
unregister(message,websocket)
async def create_peer(username,websocket):
global pcs
global peer_connections
global local_username
pc = RTCPeerConnection()
pc_index = peer_connections
pcs.append([username,pc])
peer_connections = peer_connections+1
'''
add stream (video-audio to pc)
'''
#pc.on("track")
def on_track(track):
'''
play track with PyQt5
'''
offer = await pcs[pc_index].createOffer()
pcs[pc_index][1].setLocalDescription(offer)
data = {"type":"offer","from":local_username,"to":username,"offer":offer}
await websocket.send(json.dumps(data))
async def receive_offer(message,websocket):
global pcs
global peer_connections
global local_username
username = message["username"]
offer = RTCSessionDescription(sdp=message["offer"]["sdp"], type=message["offer"]["type"])
pc = RTCPeerConnection()
pc_index = peer_connections
pcs.append([username,pc])
peer_connections = peer_connections+1
'''
add stream (video-audio to pc)
'''
#pc.on("track")
def on_track(track):
'''
play track with PyQt5
'''
await pcs[pc_index][1].setRemoteDescription(offer)
answer = await pcs[pc_index][1].createAnswer()
await pcs[pc_index][1].setLocalDescription(answer)
answer_json = {"sdp":answer.sdp,"type":"answer"}
data = {"type":"answer","from":local_username,"to":username,"answer":answer_json}
await websocket.send(json.dumps(data))
async def receive_answer(message,websocket):
global pcs
username = message["username"]
answer = message["answer"]
for pc in pcs:
if(pc[0]==username):
pc[1].setRemoteDescription(answer)
async def unregister(message,websocket):
global pcs
global peer_connections
username = message["username"]
index = 0
counter = 0
for pc in pcs:
if(pc[0]==username):
pc[1].close()
'''
PyQt5 manage for closing Output Device
'''
index = counter
break
counter = counter + 1
del pcs[index]
peer_connections = peer_connections-1
asyncio.get_event_loop().run_until_complete(open_websocket())
websocket_server.py
import asyncio
import websockets
import json
import ssl
peers = ()
async def on_open(websocket,path):
async for message in websocket:
message = json.loads(message)
if(message["type"]=="register"):
await register(websocket,message["username"])
elif(message["type"]=="offer"):
await send_offer(websocket,message)
elif(message["type"]=="answer"):
await send_answer(websocket,message)
elif(message["type"]=="candidate"):
await send_candidate(websocket,message)
await unregister(websocket)
async def register(websocket,username):
global peers
print(username+" logged in.")
peers = peers + ((websocket,username),)
for peer in peers:
if peer[0] is not websocket:
await websocket.send(json.dumps({"type": "create_peer","username":peer[1]}))
async def send_offer(websocket,message):
global peers
offer_creator = message["from"]
offer_receiver = message["to"]
offer = message["offer"]
print(offer_creator+" creates and sends offer to "+offer_receiver)
for peer in peers:
if(peer[1]==offer_receiver):
await peer[0].send(json.dumps({"type": "offer","username":offer_creator,"offer":offer}))
async def send_answer(websocket,message):
global peers
answer_creator = message["from"]
answer_receiver = message["to"]
answer = message["answer"]
print(answer_creator+" creates and sends answer to "+answer_receiver)
for peer in peers:
if(peer[1]==answer_receiver):
await peer[0].send(json.dumps({"type": "answer","username":answer_creator,"answer":answer}))
async def send_candidate(websocket,message):
global peers
candidate_creator = message["from"]
candidate_receiver = message["to"]
candidate = message["candidate"]
print(candidate_creator+" send candidate packet to "+candidate_receiver)
for peer in peers:
if(peer[1]==candidate_receiver):
await peer[0].send(json.dumps({"type": "candidate","username":candidate_creator,"candidate":candidate}))
async def unregister(websocket):
global peers
for peer_1 in peers:
if(peer_1[0]==websocket):
username = peer_1[1]
print(username+" logged out.")
for peer_2 in peers:
if(peer_2[0] is not websocket):
await peer_2[0].send(json.dumps({"type": "unregister","username":username}))
peers_list = list(peers)
peers_list.remove((websocket,username))
peers = tuple(peers_list)
ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
ssl_context.load_cert_chain(r'C:\xampp\crt\localhost\server.crt',r'C:\xampp\crt\localhost\server.key')
ssl_context = None
start_server = websockets.serve(on_open, "192.168.1.5", 8080, ssl=ssl_context)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()
First i run the websocket_server.py (this code is handling the websocket requests)
Second i run the websocket_client.py
After that i saw in the first cmd: epalxeis logged in.
So the connection is correct.
But if i close the second cmd (with the x button or with the ctr-c), i saw this error in the server's cmd:
Error in connection handler
Traceback (most recent call last):
File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\asyncio\windows_events.py", line 453, in finish_recv
return ov.getresult()
OSError: [WinError 64] Το καθορισμένο όνομα δικτύου δεν είναι πια διαθέσιμο
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\site-packages\websockets\protocol.py", line 827, in transfer_data
message = await self.read_message()
File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\site-packages\websockets\protocol.py", line 895, in read_message
frame = await self.read_data_frame(max_size=self.max_size)
File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\site-packages\websockets\protocol.py", line 971, in read_data_frame
frame = await self.read_frame(max_size)
File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\site-packages\websockets\protocol.py", line 1047, in read_frame
frame = await Frame.read(
File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\site-packages\websockets\framing.py", line 105, in read
data = await reader(2)
File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\asyncio\streams.py", line 723, in readexactly
await self._wait_for_data('readexactly')
File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\asyncio\streams.py", line 517, in _wait_for_data
await self._waiter
File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\asyncio\proactor_events.py", line 280, in _loop_reading
data = fut.result()
File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\asyncio\windows_events.py", line 808, in _poll
value = callback(transferred, key, ov)
File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\asyncio\windows_events.py", line 457, in finish_recv
raise ConnectionResetError(*exc.args)
ConnectionResetError: [WinError 64] Το καθορισμένο όνομα δικτύου δεν είναι πια διαθέσιμο
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\site-packages\websockets\server.py", line 191, in handler
await self.ws_handler(self, path)
File "websocket_server.py", line 9, in on_open
async for message in websocket:
File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\site-packages\websockets\protocol.py", line 439, in __aiter__
yield await self.recv()
File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\site-packages\websockets\protocol.py", line 509, in recv
await self.ensure_open()
File "C:\Users\Χρήστος\AppData\Local\Programs\Python\Python38\lib\site-packages\websockets\protocol.py", line 803, in ensure_open
raise self.connection_closed_exc()
websockets.exceptions.ConnectionClosedError: code = 1006 (connection closed abnormally [internal]), no reason
The proccess don't stop.
How can i resolve the issue?
Thanks in advance,
Chris Pappas
This post helps you.
The reason program couldn't stop is that sub threads continue to run even if main thread catches an exception and stop. Note that asyncio is implemented by thread (I pursued codes). There are two kinds of thread in Python, normal thread and daemon thread. The former continues to run if parent thread dies, the latter stops. asyncio is implemented with the former, therefore such this problem happens.

discord.py Invite Tracker

When i use a script i found on here, which allowed you to track a users invites another member
to discord. The script works great, for a short while then i get the following error
Task exception was never retrieved
future: exception=AttributeError("'NoneType' object has no attribute 'invites'")>
Traceback (most recent call last):
File "H:/TheVoidv2/bot.py", line 1509, in fetch
invs = await gld.invites()
and this error once the bot has ran for a good few hours
Task exception was never retrieved
future: <Task finished name='Task-1' coro=<fetch() done, defined at C:\Users\Administrator\Desktop\TheVoidv2\bot.py:1416> exception=HTTPException('503 Service Unavailable (error code: 0): upstream connect error or disconnect/reset before headers. reset reason: connection failure')>
Traceback (most recent call last):
File "C:\Users\Administrator\Desktop\TheVoidv2\bot.py", line 1423, in fetch
invs = await gld.invites()
File "C:\Users\Administrator\AppData\Local\Programs\Python\Python38\lib\site-packages\discord\guild.py", line 1407, in invites
data = await self._state.http.invites_from(self.id)
File "C:\Users\Administrator\AppData\Local\Programs\Python\Python38\lib\site-packages\discord\http.py", line 225, in request
raise HTTPException(r, data)
discord.errors.HTTPException: 503 Service Unavailable (error code: 0): upstream connect error or disconnect/reset before headers. reset reason: connection failure
import datetime
import json
import os
import commands
client = discord.Client()
cfg = open("config.json", "r")
tmpconfig = cfg.read()
cfg.close()
config = json.loads(tmpconfig)
token = config["token"]
guild_id = config["server-id"]
logs_channel = config["logs-channel-id"]
invites = {}
last = ""
async def fetch():
global last
global invites
await client.wait_until_ready()
gld = client.get_guild(int(guild_id))
logs = client.get_channel(int(logs_channel))
while True:
invs = await gld.invites()
tmp = []
for i in invs:
for s in invites:
if s[0] == i.code:
if int(i.uses) > s[1]:
usr = gld.get_member(int(last))
testh = f"{usr.name} **joined**; Invited by **{i.inviter.name}** (**{str(i.uses)}** invites)"
await logs.send(testh)
tmp.append(tuple((i.code, i.uses)))
invites = tmp
await asyncio.sleep(4)
#client.event
async def on_ready():
print("ready!")
await client.change_presence(activity = discord.Activity(name = "joins", type = 2))
#client.event
async def on_member_join(meme):
global last
last = str(meme.id)
client.loop.create_task(fetch())
client.run(token)

How to prevent raise asyncio.TimeoutError and continue the loop

I'm using aiohttp with limited_as_completed method to speed up scrapping (around 100 million static website pages). However, the code stops after several minutes, and returns the TimeoutError. I tried several things, but still could not prevent the raise asyncio.TimeoutError. May I ask how can I ignore the error, and continue?
The code I'm running is:
N=123
import html
from lxml import etree
import requests
import asyncio
import aiohttp
from aiohttp import ClientSession, TCPConnector
import pandas as pd
import re
import csv
import time
from itertools import islice
import sys
from contextlib import suppress
start = time.time()
data = {}
data['name'] = []
filename = "C:\\Users\\xxxx"+ str(N) + ".csv"
def limited_as_completed(coros, limit):
futures = [
asyncio.ensure_future(c)
for c in islice(coros, 0, limit)
]
async def first_to_finish():
while True:
await asyncio.sleep(0)
for f in futures:
if f.done():
futures.remove(f)
try:
newf = next(coros)
futures.append(
asyncio.ensure_future(newf))
except StopIteration as e:
pass
return f.result()
while len(futures) > 0:
yield first_to_finish()
async def get_info_byid(i, url, session):
async with session.get(url,timeout=20) as resp:
print(url)
with suppress(asyncio.TimeoutError):
r = await resp.text()
name = etree.HTML(r).xpath('//h2[starts-with(text(),"Customer Name")]/text()')
data['name'].append(name)
dataframe = pd.DataFrame(data)
dataframe.to_csv(filename, index=False, sep='|')
limit = 1000
async def print_when_done(tasks):
for res in limited_as_completed(tasks, limit):
await res
url = "http://xxx.{}.html"
loop = asyncio.get_event_loop()
async def main():
connector = TCPConnector(limit=10)
async with ClientSession(connector=connector,headers=headers,raise_for_status=False) as session:
coros = (get_info_byid(i, url.format(i), session) for i in range(N,N+1000000))
await print_when_done(coros)
loop.run_until_complete(main())
loop.close()
print("took", time.time() - start, "seconds.")
The error log is:
Traceback (most recent call last):
File "C:\Users\xxx.py", line 111, in <module>
loop.run_until_complete(main())
File "C:\Users\xx\AppData\Local\Programs\Python\Python37-32\lib\asyncio\base_events.py", line 573, in run_until_complete
return future.result()
File "C:\Users\xxx.py", line 109, in main
await print_when_done(coros)
File "C:\Users\xxx.py", line 98, in print_when_done
await res
File "C:\Users\xxx.py", line 60, in first_to_finish
return f.result()
File "C:\Users\xxx.py", line 65, in get_info_byid
async with session.get(url,timeout=20) as resp:
File "C:\Users\xx\AppData\Local\Programs\Python\Python37-32\lib\site-packages\aiohttp\client.py", line 855, in __aenter__
self._resp = await self._coro
File "C:\Users\xx\AppData\Local\Programs\Python\Python37-32\lib\site-packages\aiohttp\client.py", line 391, in _request
await resp.start(conn)
File "C:\Users\xx\AppData\Local\Programs\Python\Python37-32\lib\site-packages\aiohttp\client_reqrep.py", line 770, in start
self._continue = None
File "C:\Users\xx\AppData\Local\Programs\Python\Python37-32\lib\site-packages\aiohttp\helpers.py", line 673, in __exit__
raise asyncio.TimeoutError from None
concurrent.futures._base.TimeoutError
I have tried
1) add expect asyncio.TimeoutError: pass. Not working
async def get_info_byid(i, url, session):
async with session.get(url,timeout=20) as resp:
print(url)
try:
r = await resp.text()
name = etree.HTML(r).xpath('//h2[starts-with(text(),"Customer Name")]/text()')
data['name'].append(name)
dataframe = pd.DataFrame(data)
dataframe.to_csv(filename, index=False, sep='|')
except asyncio.TimeoutError:
pass
2) suppress(asyncio.TimeoutError)as shown above. Not working
I just learned aiohttp yesterday, so maybe there is other things wrong in my code that causes timeout error only after a few minutes' running? Thank you very much if anyone knows how to deal with it!
what #Yurii Kramarenko has done will raise Unclosed client session excecption for sure, since the session has never be properly closed. What I recommend is sth like this:
import asyncio
import aiohttp
async def main(urls):
async with aiohttp.ClientSession(timeout=self.timeout) as session:
tasks=[self.do_something(session,url) for url in urls]
await asyncio.gather(*tasks)
I like #jbxiaoyu answer, but the timeout kwarg seems to take a special object, so I thought I'd add you need to create a ClientTimeout object, then pass it to the Session, like this:
from aiohttp import ClientSession, ClientTimeout
timeout = ClientTimeout(total=600)
async with ClientSession(timeout=timeout) as session:
tasks=[self.do_something(session,url) for url in urls]
await asyncio.gather(*tasks)
Simple example (not very good, but works fine):
import asyncio
from aiohttp.client import ClientSession
class Wrapper:
def __init__(self, session):
self._session = session
async def get(self, url):
try:
async with self._session.get(url, timeout=20) as resp:
return await resp.text()
except Exception as e:
print(e)
loop = asyncio.get_event_loop()
wrapper = Wrapper(ClientSession())
responses = loop.run_until_complete(
asyncio.gather(
wrapper.get('http://google.com'),
wrapper.get('http://google.com'),
wrapper.get('http://google.com'),
wrapper.get('http://google.com'),
wrapper.get('http://google.com')
)
)
print(responses)

Categories