This is a simple program to retrieve candlestick data from Binance exchange for several pairs. I found out that it could be done with asyncio package.
import websockets
import asyncio
import json
import pprint
async def candle_stick_data():
url = "wss://stream.binance.com:9443/ws/" #steam address
first_pair = 'xlmbusd#kline_1m' #first pair
async with websockets.connect(url+first_pair) as sock:
pairs = '{"method": "SUBSCRIBE", "params": ["xlmbnb#kline_1m","bnbbusd#kline_1m" ], "id": 1}' #other pairs
await sock.send(pairs)
print(f"> {pairs}")
while True:
resp = await sock.recv()
resp=json.loads(resp)
pprint.pprint(resp)
candle = resp['k']
asyncio.get_event_loop().run_until_complete(candle_stick_data())
I am getting messages and changing type to dict with json.loads(resp). My question is how can I access dict values because candle = resp['k'] causes "Key error 'k'". I am new to asyncio maybe I don't need it at all to retrieve data for several pairs.
updated message screenshot
Your first incoming message really does not have 'k' key in dictionary.
I just added if else block to your code and it works well:
import websockets
import asyncio
import json
import pprint
async def candle_stick_data():
url = "wss://stream.binance.com:9443/ws/" #steam address
first_pair = 'xlmbusd#kline_1m' #first pair
async with websockets.connect(url+first_pair) as sock:
pairs = '{"method": "SUBSCRIBE", "params": ["xlmbnb#kline_1m","bnbbusd#kline_1m" ], "id": 1}' #other pairs
await sock.send(pairs)
print(f"> {pairs}")
while True:
resp = await sock.recv()
resp = json.loads(resp)
# get 'k' key value if it exits, otherwise None
k_key_val = resp.get('k', None)
# easy if else block
if not k_key_val:
print(f"No k key found: {resp}")
else:
pprint.pprint(k_key_val)
if __name__ == '__main__':
asyncio.get_event_loop().run_until_complete(candle_stick_data())
Related
My originale requeste is:
def get_foo_by_bars(authorisation_token: str, bar_ids: list):
r = requests.get(BASE_URL + "/api/v1/foo/bar",
params={"bar_ids": bar_ids, "data_type": "Float"},
headers={"Authorization": authorisation_token})
if r.status_code == 200:
return r.json()["data"]["data"]
My problem is bar_ids size contain more 80 element so my url size is more 2048 char. I want to be able to launch several requests in parallel with for example 10 bar_id then do a merge of the x responses at the end before the return.
That might be possible via asyncio + aiohttp. Unfortunatly I have no API to test this against right now, so the following code might have some issues, but should at least give you an idea:
import asyncio
import aiohttp
# async function to get json result for subset of bar_ids
async def get(session, **kwargs):
try:
async with session.get(**kwargs) as response:
await response.read()
if response.status == 200:
return await response.json()
return {}
except Exception as exc:
print(f"ERROR:\n{kwargs}\n{exc}")
return {}
# async function to split bar_ids into subsets, get their result and join them to the final result
async def main(bar_ids, package_size):
async with aiohttp.ClientSession() as session:
packaged_kwargs = [{
"url": BASE_URL + "/api/v1/foo/bar",
"params": {"bar_ids": bar_ids[i:i + package_size], "data_type": "Float"},
"headers": {"Authorization": AUTHORIZATION_TOKEN},
} for i in range(0, len(bar_ids), package_size)]
json_list = await asyncio.gather(*[get(session, **kwargs) for kwargs in packaged_kwargs])
result = {key: value for json_dict in json_list for key, value in json_dict.items()}
print(result)
# parameters
BASE_URL = "https://www.google.com"
AUTHORIZATION_TOKEN = "823ljf9823puj8รถ3"
bar_ids = list(range(100))
package_size = 10
# run
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) # only required for windows
asyncio.run(main(bar_ids,package_size))
I have a huge list of urls that I need to send request and retrieve a json data.But the problem is Since the list with the urls is too big to load it at once, I would like to read the urls one by one, and each time the url is loaded, it should start a request. My code work for small list(~20k) with no problem but I got stuck with a huge list.
It would be great if you could tell me how to change my code, to get it to send asynchronous requests for each url of the urls list. Thank you in advance.
Here is my code:
import json
import urllib
from urllib.parse import quote
import time
import asyncio
import aiohttp
import json
from json.decoder import JSONDecodeError
urls = ["url_1", "url_2". "url_3"........"url_3,000,000"]
START = time.monotonic()
class RateLimiter:
RATE = 20
MAX_TOKENS = 10
def __init__(self, client):
self.client = client
self.tokens = self.MAX_TOKENS
self.updated_at = time.monotonic()
async def get(self, *args, **kwargs):
await self.wait_for_token()
now = time.monotonic() - START
print(f'{now:.0f}s: ask {args[0]}')
return self.client.get(*args, **kwargs)
async def wait_for_token(self):
while self.tokens < 1:
self.add_new_tokens()
await asyncio.sleep(0.1)
self.tokens -= 1
def add_new_tokens(self):
now = time.monotonic()
time_since_update = now - self.updated_at
new_tokens = time_since_update * self.RATE
if self.tokens + new_tokens >= 1:
self.tokens = min(self.tokens + new_tokens, self.MAX_TOKENS)
self.updated_at = now
async def fetch_one(client, url):
# Watch out for the extra 'await' here!
async with await client.get(url) as resp:
for response in resp:
try:
results = await response.json()
try:
answer = results['results'][0]['locations']
output = {
"Provided location" : results['results'][0]['providedLocation'].get('location'),
"City": answer[0].get('adminArea5'),
"State" : answer[0].get('adminArea3'),
"Country": answer[0].get('adminArea1')
}
json_results.append(output)
except (IndexError,JSONDecodeError):
output = {
"Provided location": 'null',
"City": 'null',
"State" : 'null',
"Country":'null'
}
json_results.append(output)
except:
output = {
"Provided location": None,
"City": 'null',
"State" : 'null',
"Country":'null'
}
json_results.append(output)
now = time.monotonic() - START
async def main():
async with aiohttp.ClientSession() as client:
client = RateLimiter(client)
tasks = [asyncio.ensure_future(fetch_one(client, url)) for url in urls]
await asyncio.gather(*tasks)
if __name__ == '__main__':
asyncio.run(main())
I've successfully connected to the Bitstamp websocket and I am now attempting to index the data stream that I receive. Specifically, I want to save bids[0][0] into a best_price variable:
{'data': {'timestamp': '1615553987', 'microtimestamp': '1615553987492634', 'bids': [['56355.57', '0.09439734'], ['56347.20', '0.03743896'], ['56346.03', '0.47172493']....etc
The problem is that I get a "subscription succeeded" message when I first connect:
{'event': 'bts:subscription_succeeded', 'channel': 'order_book_btcusd', 'data': {}}
This means I get the following error, because I cannot index the None returned from the empty data steam:
IndexError: too many indices for array: array is 0-dimensional, but 1 were indexed
Here is my code
import asyncio
import websockets
import json
import numpy as np
def error_handler(err):
pass
def handler(msg):
pass
async def bitstamp_connect(callback):
uri = "wss://ws.bitstamp.net/"
subscription = {
"event": "bts:subscribe",
"data": {
"channel": "order_book_btcusd"
}
}
async with websockets.connect(uri) as websocket:
await websocket.send(json.dumps(subscription))
while True:
msg = json.loads(await websocket.recv())
bids = np.array(msg['data'].get('bids'))
#print(bids)
print(bids[0][0])
asyncio.get_event_loop().run_until_complete(bitstamp_connect(handler))
The problem can be easily solved by simply adding an if statement checking if the message received was a 'subscription successful' message or a 'data' message. Change your while loop to the following:
while True:
msg = json.loads(await websocket.recv())
# Check what type of message we received
if msg['event'] == 'data':
bids = np.array(msg['data'].get('bids'))
print(bids[0][0])
elif msg['event'] == 'bts:subscription_succeeded':
# You can put code here if you want to do something when first subscribing
pass
So, I was making a Discord bot using discord.py and here's my code:
#bot.command()
async def list(ctx):
ok = 0
for i in db.keys():
# lmao why is there even a while loop here #
while ok >= 11:
server_num = ok+1
em = discord.Embed(title='Oldest servers in the list!',description=f'Server {server_num}')
em.add_field(name='Name',value=db[i]['name'],inline=False)
em.add_field(name='Description',value=db[i]['description'],inline=False)
em.add_field(name='Tags',value=db[i]['tags'],inline=False)
await ctx.send(embed=em)
ok+=1
But, what I also want is that it returns the servers with most number of upvotes, we have an upvotes system and the code for it is:
#bot.command()
#commands.cooldown(1,7200)
async def upvote(ctx):
try:
id = str(ctx.message.guild.id)
if id in db.keys():
db[id]["upvote"][0] += 1
await ctx.send(f"upvoted ")
save()
except commands.CommandOnCooldown:
await ctx.send('once every 12 hours ;---------------;')
except:
await ctx.send('error ;-;')
And, every upvote is stored in a JSON file (db.json) which looks like this:
{
"738049816357109831": {
"description": "I am a good server",
"id": "738049816357109831",
"link": "https://discord.gg/gQ6FABc",
"name": "Aypro's Lab",
"review(s)": [],
"tags": [
"#good_boi"
],
"upvote": [
0
]
}
}
And I am not really sure how to get the servers with most number of up votes. Thanks to who ever will answer this as it is going to really help me with my bots development.
A simple way that I'd do it would be to incorporate a function to create a temporarily restructured dictionary that allows for you to easily get the max key from your json db.
def get_highest_server(dictionary):
server_votes = {}
for key, value in dictionary.items():
server_votes[key] = dictionary[key]['upvote']
return max(server_votes, key=server_votes.get)
This will iterate through your DB file and restructure it into a dictionary that has only 1 key and one value (the value being the number of upvotes). A demonstration on using this function is as below:
import json
def get_highest_server(dictionary):
server_votes = {}
for key, value in dictionary.items():
server_votes[key] = dictionary[key]['upvote']
return max(server_votes, key=server_votes.get)
with open("db.json", "r") as dFile:
databases = json.load(dFile)
highest_server = get_highest_server(databases)
Like title told, my use case is like this:
I have one aiohttp server, which accept request from client, when i have the request i generate one unique request id for it, and then i send the {req_id: req_pyaload} dict to some workers (the worker is not in python thus running in another process), when the workers complete the work, i get back the response and put them in a result dict like this: {req_id_1: res_1, req_id_2: res_2}.
Then I want my aiohttp server handler to await on above result dict, so when the specific response become available (by req_id) it can send it back.
I build below example code to try to simulate the process, but got stuck in implementing the coroutine async def fetch_correct_res(req_id) which should asynchronously/unblockly fetch the correct response by req_id.
import random
import asyncio
import shortuuid
n_tests = 1000
idxs = list(range(n_tests))
req_ids = []
for _ in range(n_tests):
req_ids.append(shortuuid.uuid())
res_dict = {}
async def fetch_correct_res(req_id):
pass
async def handler(req):
res = await fetch_correct_res(req)
assert req == res, "the correct res for the req should exactly be the req itself."
print("got correct res for req: {}".format(req))
async def randomly_put_res_to_res_dict():
for _ in range(n_tests):
random_idx = random.choice(idxs)
await asyncio.sleep(random_idx / 1000)
res_dict[req_ids[random_idx]] = req_ids[random_idx]
print("req: {} is back".format(req_ids[random_idx]))
So:
Is it possible to make this solution work? how?
If above solution is not possible, what should be the correct solution for this use case with asyncio?
Many thanks.
The only approach i can think of for now to make this work is: pre-created some asyncio.Queue with pre-assigned id, then for each incoming request assign one queue to it, so the handler just await on this queue, when the response come back i put it into this pre-assigned queue only, after the request fulfilled, i collect back the queue to use it for next incoming request. Not very elegant, but will solve the problem.
See if the below sample implementation fulfils your need
basically you want to respond back to the request(id) with your response(unable to predict the order) in an asynchronous way
So at the time of request handling, populate the dict with {request_id: {'event':<async.Event>, 'result': <result>}} and await on asyncio.Event.wait(), once the response is received, signal the event with asyncio.Event.set() which will release the await and then fetch the response from the dict based on the request id
I modified your code slightly to pre-populate the dict with request id and put the await on asyncio.Event.wait() until the signal comes from the response
import random
import asyncio
import shortuuid
n_tests = 10
idxs = list(range(n_tests))
req_ids = []
for _ in range(n_tests):
req_ids.append(shortuuid.uuid())
res_dict = {}
async def fetch_correct_res(req_id, event):
await event.wait()
res = res_dict[req_id]['result']
return res
async def handler(req, loop):
print("incoming request id: {}".format(req))
event = asyncio.Event()
data = {req :{}}
res_dict.update(data)
res_dict[req]['event']=event
res_dict[req]['result']='pending'
res = await fetch_correct_res(req, event)
assert req == res, "the correct res for the req should exactly be the req itself."
print("got correct res for req: {}".format(req))
async def randomly_put_res_to_res_dict():
random.shuffle(req_ids)
for i in req_ids:
await asyncio.sleep(random.randrange(2,4))
print("req: {} is back".format(i))
if res_dict.get(i) is not None:
event = res_dict[i]['event']
res_dict[i]['result'] = i
event.set()
loop = asyncio.get_event_loop()
tasks = asyncio.gather(handler(req_ids[0], loop),
handler(req_ids[1], loop),
handler(req_ids[2], loop),
handler(req_ids[3], loop),
randomly_put_res_to_res_dict())
loop.run_until_complete(tasks)
loop.close()
sample response from the above code
incoming request id: NDhvBPqMiRbteFD5WqiLFE
incoming request id: fpmk8yC3iQcgHAJBKqe2zh
incoming request id: M7eX7qeVQfWCCBnP4FbRtK
incoming request id: v2hAfcCEhRPUDUjCabk45N
req: VeyvAEX7YGgRZDHqa2UGYc is back
req: M7eX7qeVQfWCCBnP4FbRtK is back
got correct res for req: M7eX7qeVQfWCCBnP4FbRtK
req: pVvYoyAzvK8VYaHfrFA9SB is back
req: soP8NDxeQKYjgeT7pa3wtG is back
req: j3rcg5Lp59pQXuvdjCAyZe is back
req: NDhvBPqMiRbteFD5WqiLFE is back
got correct res for req: NDhvBPqMiRbteFD5WqiLFE
req: v2hAfcCEhRPUDUjCabk45N is back
got correct res for req: v2hAfcCEhRPUDUjCabk45N
req: porzHqMqV8SAuttteHRwNL is back
req: trVVqZrUpsW3tfjQajJfb7 is back
req: fpmk8yC3iQcgHAJBKqe2zh is back
got correct res for req: fpmk8yC3iQcgHAJBKqe2zh
This may work (note: I removed UUID in order to know req id in advance)
import random
import asyncio
n_tests = 1000
idxs = list(range(n_tests))
req_ids = []
for i in range(n_tests):
req_ids.append(i)
res_dict = {}
async def fetch_correct_res(req_id):
while not res_dict.get(req_id):
await asyncio.sleep(0.1)
return req_ids[req_id]
async def handler(req):
print("fetching req: ", req)
res = await fetch_correct_res(req)
assert req == res, "the correct res for the req should exactly be the req itself."
print("got correct res for req: {}".format(req))
async def randomly_put_res_to_res_dict(future):
for i in range(n_tests):
res_dict[req_ids[i]] = req_ids[i]
await asyncio.sleep(0.5)
print("req: {} is back".format(req_ids[i]))
future.set_result("done")
loop = asyncio.get_event_loop()
future = asyncio.Future()
asyncio.ensure_future(randomly_put_res_to_res_dict(future))
loop.run_until_complete(handler(10))
loop.close()
Is it the best solution? according to me No, basically its kind of requesting long running job status, and you should have (REST) api for doing the job submission and knowing job status like:
http POST server:port/job
{some job json paylod}
Response: 200 OK {"req_id": 1}
http GET server:port/job/1
Response: 200 OK {"req_id": 1, "status": "in process"}
http GET server:port/job/1
Response: 200 OK {"req_id": 1, "status": "done", "result":{}}