We Keep Coding

Tornado gives error Cannot write() after finish()

I am using the Tornado chat demo example from here: https://github.com/tornadoweb/tornado/tree/master/demos/chat
and just altering it very slightly.
The code change is just a small class called Connections and a bit in the MessageNewHandler(). All I am doing is just saving a reference to self and trying to write(message) to a previous client.
But when I go to save on this line conns.conns[0].write(message) I get this error message:
[E 220107 23:18:38 web:1789] Uncaught exception POST /a/message/new (::1)
HTTPServerRequest(protocol='http', host='localhost:8888', method='POST', uri='/a/message/new', version='HTTP/1.1', remote_ip='::1')
Traceback (most recent call last):
File "/home/joe/dev/tornado/lib/python3.8/site-packages/tornado/web.py", line 1702, in _execute
result = method(*self.path_args, **self.path_kwargs)
File "server.py", line 89, in post
MessageNewHandler.clients[0].write(message)
File "/home/joe/dev/tornado/lib/python3.8/site-packages/tornado/web.py", line 833, in write
raise RuntimeError("Cannot write() after finish()")
RuntimeError: Cannot write() after finish()
[E 220107 23:18:38 web:2239] 500 POST /a/message/new (::1) 5.98ms
Here is the code:
import asyncio
import tornado.escape
import tornado.ioloop
import tornado.locks
import tornado.web
import os.path
import uuid
from tornado.options import define, options, parse_command_line
define("port", default=8888, help="run on the given port", type=int)
define("debug", default=True, help="run in debug mode")
class Connections(object):
def __init__(self):
self.conns = []
def add_connection(self, conn_self):
self.conns.append(conn_self)
def conns(self):
return self.conns
conns = Connections()
class MessageBuffer(object):
def __init__(self):
# cond is notified whenever the message cache is updated
self.cond = tornado.locks.Condition()
self.cache = []
self.cache_size = 200
def get_messages_since(self, cursor):
"""Returns a list of messages newer than the given cursor.
``cursor`` should be the ``id`` of the last message received.
"""
results = []
for msg in reversed(self.cache):
if msg["id"] == cursor:
break
results.append(msg)
results.reverse()
return results
def add_message(self, message):
self.cache.append(message)
if len(self.cache) > self.cache_size:
self.cache = self.cache[-self.cache_size :]
self.cond.notify_all()
# Making this a non-singleton is left as an exercise for the reader.
global_message_buffer = MessageBuffer()
class MainHandler(tornado.web.RequestHandler):
def get(self):
self.render("index.html", messages=global_message_buffer.cache)
class MessageNewHandler(tornado.web.RequestHandler):
"""Post a new message to the chat room."""
def post(self):
message = {"id": str(uuid.uuid4()), "body": self.get_argument("body")}
# render_string() returns a byte string, which is not supported
# in json, so we must convert it to a character string.
message["html"] = tornado.escape.to_unicode(
self.render_string("message.html", message=message)
)
conns.add_connection(self)
if (len(conns.conns)>2):
conns.conns[0].write(message)
self.finish()
class MessageUpdatesHandler(tornado.web.RequestHandler):
"""Long-polling request for new messages.
Waits until new messages are available before returning anything.
"""
async def post(self):
cursor = self.get_argument("cursor", None)
messages = global_message_buffer.get_messages_since(cursor)
while not messages:
# Save the Future returned here so we can cancel it in
# on_connection_close.
self.wait_future = global_message_buffer.cond.wait()
try:
await self.wait_future
except asyncio.CancelledError:
return
messages = global_message_buffer.get_messages_since(cursor)
if self.request.connection.stream.closed():
return
self.write(dict(messages=messages))
def on_connection_close(self):
self.wait_future.cancel()
def main():
parse_command_line()
app = tornado.web.Application(
[
(r"/", MainHandler),
(r"/a/message/new", MessageNewHandler),
(r"/a/message/updates", MessageUpdatesHandler),
],
cookie_secret="__TODO:_GENERATE_YOUR_OWN_RANDOM_VALUE_HERE__",
template_path=os.path.join(os.path.dirname(__file__), "templates"),
static_path=os.path.join(os.path.dirname(__file__), "static"),
xsrf_cookies=True,
debug=options.debug,
)
app.listen(options.port)
tornado.ioloop.IOLoop.current().start()
if __name__ == "__main__":
main()

You're writing to an already closed connection, that's why you're seeing the error.
If you want to write to a previously connected client, you've keep that connection open.
However, this - conns.add_connection(self) - doesn't make sense to track regular http connections.
You should consider using websockets if you want to keep previous connections open and track them.
Update: Here's how you can keep a connection open. If I understand correctly, you want to send a message from current client to the previous client.
1. Using tornado.locks.Condition():
import tornado.locks
class MessageNewHandler(tornado.web.RequestHandler):
"""Post a new message to the chat room."""
clients = []
condition = tornado.locks.Condition()
async def post(self):
message = {"id": str(uuid.uuid4()), "body": self.get_argument("body")}
# render_string() returns a byte string, which is not supported
# in json, so we must convert it to a character string.
message["html"] = tornado.escape.to_unicode(
self.render_string("message.html", message=message)
)
MessageNewHandler.clients.append(self)
if len(MessageNewHandler.clients) < 2:
# less than 2 clients
# wait until notified
await MessageNewHandler.condition.wait()
else:
# at least 2 clients
# write to previous client's response
MessageNewHandler.clients[0].finish(message)
# notify the first waiting client
# so it can send the response
MessageNewHandler.condition.notify()
# Note: since you've finished previous client's response
# you should also remove it from clients list
# since you can't use that connection again
2. Using tornado.concurrent.Future():
import tornado.concurrent
class MessageNewHandler(tornado.web.RequestHandler):
"""Post a new message to the chat room."""
waiters = []
async def post(self):
message = {"id": str(uuid.uuid4()), "body": self.get_argument("body")}
# render_string() returns a byte string, which is not supported
# in json, so we must convert it to a character string.
message["html"] = tornado.escape.to_unicode(
self.render_string("message.html", message=message)
)
future = tornado.concurrent.Future() # create a future
# instead of a saving the reference to the client,
# save the future
MessageNewHandler.waiters.append(future)
if len(MessageNewHandler.waiters) < 2:
# less than 2 clients
# wait for next client's message
msg_from_next_client = await future
# the future will resolve when the next client
# sets a result on it
# then python will execute the following code
self.finish(msg_from_next_client)
# Note: since you've finished this connection
# you should remove this future from the waiters list
# since you can't reuse this connection again
else:
# at least 2 clients
# set the current client's message
# as a result on previous client's future
previous_client_future = MessageNewHandler.waiters[0]
if not previous_client_future.done():
# only set a result if you haven't set it already
# otherwise you'll get an error
previous_client_future.set_result(message)
3: A more practical example using tornado.concurrent.Future():
import tornado.concurrent
class Queue:
"""We'll keep the future related code in this class.
This will allow us to present a cleaner, more intuitive usage api.
"""
waiters = []
#classmethod
def get_message_from_next_client(cls):
future = tornado.concurrent.Future()
cls.waiters.append(future)
return future
#classmethod
def send_message_to_prev_client(cls, message):
previous_client_future = cls.waiters[0]
if not previous_client_future.done():
previous_client_future.set_result(message)
class MessageNewHandler(tornado.web.RequestHandler):
"""Post a new message to the chat room."""
async def post(self):
message = {"id": str(uuid.uuid4()), "body": self.get_argument("body")}
message["html"] = tornado.escape.to_unicode(
self.render_string("message.html", message=message)
)
if len(Queue.waiters) < 2:
msg_from_next_client = await Queue.get_message_from_next_client()
self.finish(msg_from_next_client)
else:
Queue.send_message_to_prev_client(message)

I had a look at the RequestHandler code on https://github.com/tornadoweb/tornado/blob/master/tornado/web.py
I got rid of the Connection class and changed the MessageNewHandler to this..
class MessageNewHandler(tornado.web.RequestHandler):
"""Post a new message to the chat room."""
clients =[]
def post(self):
self._auto_finish =False
message = {"id": str(uuid.uuid4()), "body": self.get_argument("body")}
# render_string() returns a byte string, which is not supported
# in json, so we must convert it to a character string.
message["html"] = tornado.escape.to_unicode(
self.render_string("message.html", message=message)
)
MessageNewHandler.clients.append(self)
if len(MessageNewHandler.clients)>1:
MessageNewHandler.clients[0].finish(message)
So the 2 key things that made it work were self._auto_finish =False
and MessageNewHandler.clients[0].finish(message)

Binance Websocket for futures

I can't make the websocket for COIN-M on Binance returns any data.
I managed to get it to work with the SPOT endpoint, but in the Futures it is a null response.
import websocket
import _thread
import time
import json
WEBSOCKET_URL_FUTURES = "wss://dstream.binance.com/ws"
WEBSOCKET_URL_SPOT = "wss://stream.binance.com:9443/ws"
def on_message(ws, message):
print(f'ONMESSAGE######: {message}')
with open ('./data.json', 'a') as f:
f.write(f'{message}\n')
def on_error(ws, error):
print(f'ONERROR####: {error}')
def on_close(ws, close_status_code, close_msg):
print("### closed ###")
def on_open(ws):
print("### opened ###")
subscribe_message = {
"method": "SUBSCRIBE",
"params":
[
"ethusd_perpetual#aggTrade"
],
"id": 1
}
list_message = {
"method": "LIST_SUBSCRIPTIONS",
"id": 3
}
ws.send(json.dumps(subscribe_message))
ws.send(json.dumps(list_message))
if __name__ == "__main__":
websocket.enableTrace(True)
ws = websocket.WebSocketApp(WEBSOCKET_URL_SPOT,
on_open=on_open,
on_message=on_message,
on_error=on_error,
on_close=on_close)
ws.run_forever()
I'm trying differents parameters in the SUSBCRIBE method but neither of them returns anything.
Also, I can't find any list with all the possible symbol pairs that could be used.
Any ideas?

There might be a typo in your stream URL.
The code uses wss://dstream..., while this should be wss://fstream...

How to Index Websocket Data Stream

I've successfully connected to the Bitstamp websocket and I am now attempting to index the data stream that I receive. Specifically, I want to save bids[0][0] into a best_price variable:
{'data': {'timestamp': '1615553987', 'microtimestamp': '1615553987492634', 'bids': [['56355.57', '0.09439734'], ['56347.20', '0.03743896'], ['56346.03', '0.47172493']....etc
The problem is that I get a "subscription succeeded" message when I first connect:
{'event': 'bts:subscription_succeeded', 'channel': 'order_book_btcusd', 'data': {}}
This means I get the following error, because I cannot index the None returned from the empty data steam:
IndexError: too many indices for array: array is 0-dimensional, but 1 were indexed
Here is my code
import asyncio
import websockets
import json
import numpy as np
def error_handler(err):
pass
def handler(msg):
pass
async def bitstamp_connect(callback):
uri = "wss://ws.bitstamp.net/"
subscription = {
"event": "bts:subscribe",
"data": {
"channel": "order_book_btcusd"
}
}
async with websockets.connect(uri) as websocket:
await websocket.send(json.dumps(subscription))
while True:
msg = json.loads(await websocket.recv())
bids = np.array(msg['data'].get('bids'))
#print(bids)
print(bids[0][0])
asyncio.get_event_loop().run_until_complete(bitstamp_connect(handler))

The problem can be easily solved by simply adding an if statement checking if the message received was a 'subscription successful' message or a 'data' message. Change your while loop to the following:
while True:
msg = json.loads(await websocket.recv())
# Check what type of message we received
if msg['event'] == 'data':
bids = np.array(msg['data'].get('bids'))
print(bids[0][0])
elif msg['event'] == 'bts:subscription_succeeded':
# You can put code here if you want to do something when first subscribing
pass

Why is my websocket request "unauthorized" only when written in OOP, when the same works perfectly when written with just functions?

I'm writing a Python program that does some trading automation. The API I work with is from Deribit, whose preferred transport mechanism is Websocket. I'm a complete newbie to Python's websockets and asyncio modules.
Here's the code I first wrote for authenticating my client and then sending a separate private message to get an order position from the account, written only with functions and no classes:
import asyncio
import websockets
import json
CL_ID = 'qxv0EeAu'
CL_SECRET = 't24F49ocH1_qFawiKnEyqlWF5D-haABb31O8xCQhySg'
REQ_URL = 'wss://test.deribit.com/ws/api/v2'
acc_token = ''
msg = {
"jsonrpc": "2.0",
"id": 1,
"params": {}
}
async def auth_api():
global msg
global acc_token
msg["method"] = "public/auth"
msg["params"] = {
"grant_type": "client_credentials",
"client_id": CL_ID,
"client_secret": CL_SECRET,
"scope": "session:test"
}
async with websockets.connect(REQ_URL) as websocket:
await websocket.send(json.dumps(msg))
while websocket.open:
response = await websocket.recv()
response_json = json.loads(response)
acc_token = response_json["result"]["access_token"]
return
async def get_position(websocket, instrument):
global msg
global acc_token
msg["id"] += 1
msg["method"] = "private/get_position"
msg["params"] = {
"access_token": acc_token,
"instrument_name": instrument
}
await websocket.send(json.dumps(msg))
while websocket.open:
response = await websocket.recv()
return response
async def main():
global msg
await auth_api()
async with websockets.connect(REQ_URL) as websocket:
response = await get_position(websocket, "BTC-PERPETUAL")
print(response)
asyncio.get_event_loop().run_until_complete(main())
It works perfectly fine. Here's my result:
{"jsonrpc":"2.0","id":2,"result":{"total_profit_loss":0.000209124,"size_currency":-0.017402402,"size":-150.0,"settlement_price":8649.9,"realized_profit_loss":2.67e-7,"open_orders_margin":0.0,"mark_price":8619.5,"maintenance_margin":0.000100079,"leverage":100,"kind":"future","instrument_name":"BTC-PERPETUAL","initial_margin":0.000174039,"index_price":8619.45,"floating_profit_loss":0.000061161,"estimated_liquidation_price":-14.95,"direction":"sell","delta":-0.017402402,"average_price":8724.34},"usIn":1573756522511975,"usOut":1573756522512240,"usDiff":265,"testnet":true}
I decided to rewrite it the OOP way, and here's the class I created (the file is named "Call_Deribit"):
import asyncio, websockets, json
class WSClient():
def __init__(self, key=None, secret=None, url=None):
self.api_key = key
self.api_secret = secret
self.msg = {
"jsonrpc": "2.0",
"id": 0
}
if url:
self.host = url
else:
self.host = 'wss://test.deribit.com/ws/api/v2'
async def call_api(self, msg):
async with websockets.connect(self.host) as websocket:
print("Connected to URL:", self.host)
try:
await websocket.send(msg)
while websocket.open:
response = await websocket.recv()
response_json = json.loads(response)
return response_json
except Exception as e:
return e
def request(self, method, params, session=None):
msg = self.msg
msg["id"] += 1
msg["method"] = method
msg["params"] = params
if session != None:
msg["params"]["scope": "session:{}".format(session)]
return asyncio.get_event_loop().run_until_complete(self.call_api(json.dumps(msg)))
def get_order_book(self, instrument):
method = "public/get_order_book"
params = {
"instrument_name": instrument
}
return self.request(method, params)
And here's the main file I'm accessing the class from and where I make all the requests:
import json, asyncio, websockets
from Call_Deribit import WSClient
CL_ID = 'qxv0EeAu'
CL_SECRET = 't24F49ocH1_qFawiKnEyqlWF5D-haABb31O8xCQhySg'
REQ_URL = 'wss://test.deribit.com/ws/api/v2'
method_auth = "public/auth"
params_auth = {
"grant_type": "client_credentials",
"client_id": CL_ID,
"client_secret": CL_SECRET
}
main_client = WSClient(key=CL_ID, secret=CL_SECRET, url=REQ_URL)
auth_response = main_client.request(method_auth, params_auth)
acc_token = auth_response["result"]["access_token"]
method_pos = "private/get_position"
params_pos = {
"access_token": acc_token,
"instrument_name": "BTC-PERPETUAL"
}
position = main_client.request(method_pos, params_pos)
print(position)
The first request for authentication is working this time, and I'm able to extract the access token as well, but the second private/get_position message is, for whatever reason, returning an unauthorized error.
{'jsonrpc': '2.0', 'id': 1, 'error': {'message': 'unauthorized', 'code': 13009}, 'testnet': True, 'usIn': 1573756936534405, 'usOut': 1573756936534629, 'usDiff': 224}
I've spent hours on it, and I seem to be doing exactly the same thing in the OOP version as I did on the original one. My familiarity with OOP and its concepts (such as inheritance) is limited, so I'd like to know what I'm missing here, and why my code isn't working in the OOP version, despite following the same exact workflow as in the original version.
Here's the documentation for the Deribit API: https://docs.deribit.com/v2/?python#json-rpc
Any help would be greatly appreciated.

Adding the scope under params_auth in the main file works:
params_auth = {
"grant_type": "client_credentials",
"client_id": CL_ID,
"client_secret": CL_SECRET,
"scope": "session:test"
}

Ambiguity in understanding the Websocket Python

I am trying this :
import websocket
import json
def on_open(ws):
json_data = json.dumps({'ticks':'R_100'})
ws.send(json_data)
def on_message(ws, message):
print('ticks update: %s' % message)
if __name__ == "__main__":
apiUrl = "wss://ws.binaryws.com/websockets/v3?app_id=1089"
ws = websocket.WebSocketApp(apiUrl, on_message = on_message, on_open = on_open)
ws.run_forever()
This is working fine. But the issue is when I want to have a history using the payload:
{
"ticks_history": "R_50",
"end": "latest",
"start": 1,
"style": "ticks",
"adjust_start_time": 1,
"count": 10
}
I am not understanding what exactly I needed to do to get the information as the app is running and it won't serve the request to me.
Please help me get through this.