We Keep Coding

How to split array and use several requests.get in parallel with python?

My originale requeste is:
def get_foo_by_bars(authorisation_token: str, bar_ids: list):
r = requests.get(BASE_URL + "/api/v1/foo/bar",
params={"bar_ids": bar_ids, "data_type": "Float"},
headers={"Authorization": authorisation_token})
if r.status_code == 200:
return r.json()["data"]["data"]
My problem is bar_ids size contain more 80 element so my url size is more 2048 char. I want to be able to launch several requests in parallel with for example 10 bar_id then do a merge of the x responses at the end before the return.

That might be possible via asyncio + aiohttp. Unfortunatly I have no API to test this against right now, so the following code might have some issues, but should at least give you an idea:
import asyncio
import aiohttp
# async function to get json result for subset of bar_ids
async def get(session, **kwargs):
try:
async with session.get(**kwargs) as response:
await response.read()
if response.status == 200:
return await response.json()
return {}
except Exception as exc:
print(f"ERROR:\n{kwargs}\n{exc}")
return {}
# async function to split bar_ids into subsets, get their result and join them to the final result
async def main(bar_ids, package_size):
async with aiohttp.ClientSession() as session:
packaged_kwargs = [{
"url": BASE_URL + "/api/v1/foo/bar",
"params": {"bar_ids": bar_ids[i:i + package_size], "data_type": "Float"},
"headers": {"Authorization": AUTHORIZATION_TOKEN},
} for i in range(0, len(bar_ids), package_size)]
json_list = await asyncio.gather(*[get(session, **kwargs) for kwargs in packaged_kwargs])
result = {key: value for json_dict in json_list for key, value in json_dict.items()}
print(result)
# parameters
BASE_URL = "https://www.google.com"
AUTHORIZATION_TOKEN = "823ljf9823puj8ö3"
bar_ids = list(range(100))
package_size = 10
# run
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) # only required for windows
asyncio.run(main(bar_ids,package_size))

Mocking variable and replacing it with object

I'd like to test this piece of code:
modify: UserModifyPort = _ports_.user_modify_port
#_app_.route(f"/user", methods=["POST"])
#headers_check({"Accept": "application/json", "Content-Type": "application/json"})
def create_user():
body_json = request.get_json()
body = UserCreateRequest(body_json["username"], body_json["password"])
cmd = UserCreateCmd(body.username, body.password)
# modify usage
user_id = modify.create_user(cmd)
response = UserCreateResponse(user_id)
return response.to_dict(), 201
In this test I need to mock a global variable modify and replace it with object. I've been trying to do this like that:
# TEST
#mock.patch("application.user.user_rest_adapter.modify")
def test_create_user_should_create(modify_mock, db_engine, client, user_config):
modify_mock.return_value = DatabaseUserModifyAdapter(db_engine, user_config)
response = client.post("/user", headers={"Accept": "application/json", "Content-Type": "application/json"},
json={"username": "GALJO", "password": "qwerty123"})
But it isn't executing modify.create_user() function, it just returns some weird object:
<MagicMock name='modify.create_user()' id='140375141136512'>
How can I make this function work?

I solved this issue with sort of workaround. Instead of mocking entire object I've mocked just function that I use. There is no need to use another function, because it is tested in other tests so I replaced it with constant value. I've only checked if given args are correct, everything else is other test task.
#mock.patch("application.user.user_rest_adapter.modify.create_user")
def test_create_user_should_create(create_user_mock, client):
# given
user_id = "a20d7a48-7235-489b-8552-5a081d069078"
create_user_mock.return_value = UUID(user_id)
# when
response = client.post("/user", headers={"Accept": "application/json", "Content-Type": "application/json"},
json={"username": "GALJO", "password": "qwerty123"})
# then
args = create_user_mock.call_args.args
assert args[0].username == "GALJO"
assert args[0].password == "qwerty123"
assert response.json["userID"] == user_id

Using an 'If' statement to check JSON data webhook response

I am using the requests library and made a request to the twitch api. I have filtered the data into variables but I would like to know if there was a way I could use an 'If' statement on the variables. The data is stored as JSON.
Edit: I don't get any errors but the code just doesn't run.
My code is below:
client = discord.Client()
token = open("token.txt", "r").read()
myclient = pymongo.MongoClient("mongodb url")
mydb = myclient["Cluster0"]
mycol = mydb["live"]
cursor = mycol.find({ "type": "user" }, {'_id': 0, 'twitch': 1, 'channelID': 1})
for item in cursor:
x = item.get('twitch')
channelid = item.get('channelID')
print(x)
print(channelid)
headers = {
'client-id': 'twitch client id',
'Authorization': 'twitch ouath token',
}
params = (
('query', x),
)
response = requests.get('https://api.twitch.tv/helix/search/channels', headers=headers, params=params)
final = response.json()
finali = final['data'][0]['is_live']
finale = final['data'][0]['thumbnail_url']
finaly = final['data'][0]['title']
finalo = final['data'][0]['started_at']
print(final)
# I would like the If here, Eg. If finali == "True":
async def my_background_task():
await client.wait_until_ready()
counter = 0
print("someone live")
channel = client.get_channel(channelid)
while not client.is_closed():
counter += 1
embedVar2 = discord.Embed(title="" + x + " is now live on Twitch!", description="" + finaly + "", url="https://twitch.tv/" + x + "", color=0x0C8BC2)
embedVar2.set_image(url="" + finale + "")
await channel.send("#everyone")
await channel.send(embed=embedVar2)
await asyncio.sleep(60) # task runs every 60 seconds
client.loop.create_task(my_background_task())
client.run('token')

Finali should return a bool from that json so rather than testing if finali == "True" you'd just check: if finali: which is if finali == True but shorter.
So the answer is yes you can use an if statement with those variables.

Why is my websocket request "unauthorized" only when written in OOP, when the same works perfectly when written with just functions?

I'm writing a Python program that does some trading automation. The API I work with is from Deribit, whose preferred transport mechanism is Websocket. I'm a complete newbie to Python's websockets and asyncio modules.
Here's the code I first wrote for authenticating my client and then sending a separate private message to get an order position from the account, written only with functions and no classes:
import asyncio
import websockets
import json
CL_ID = 'qxv0EeAu'
CL_SECRET = 't24F49ocH1_qFawiKnEyqlWF5D-haABb31O8xCQhySg'
REQ_URL = 'wss://test.deribit.com/ws/api/v2'
acc_token = ''
msg = {
"jsonrpc": "2.0",
"id": 1,
"params": {}
}
async def auth_api():
global msg
global acc_token
msg["method"] = "public/auth"
msg["params"] = {
"grant_type": "client_credentials",
"client_id": CL_ID,
"client_secret": CL_SECRET,
"scope": "session:test"
}
async with websockets.connect(REQ_URL) as websocket:
await websocket.send(json.dumps(msg))
while websocket.open:
response = await websocket.recv()
response_json = json.loads(response)
acc_token = response_json["result"]["access_token"]
return
async def get_position(websocket, instrument):
global msg
global acc_token
msg["id"] += 1
msg["method"] = "private/get_position"
msg["params"] = {
"access_token": acc_token,
"instrument_name": instrument
}
await websocket.send(json.dumps(msg))
while websocket.open:
response = await websocket.recv()
return response
async def main():
global msg
await auth_api()
async with websockets.connect(REQ_URL) as websocket:
response = await get_position(websocket, "BTC-PERPETUAL")
print(response)
asyncio.get_event_loop().run_until_complete(main())
It works perfectly fine. Here's my result:
{"jsonrpc":"2.0","id":2,"result":{"total_profit_loss":0.000209124,"size_currency":-0.017402402,"size":-150.0,"settlement_price":8649.9,"realized_profit_loss":2.67e-7,"open_orders_margin":0.0,"mark_price":8619.5,"maintenance_margin":0.000100079,"leverage":100,"kind":"future","instrument_name":"BTC-PERPETUAL","initial_margin":0.000174039,"index_price":8619.45,"floating_profit_loss":0.000061161,"estimated_liquidation_price":-14.95,"direction":"sell","delta":-0.017402402,"average_price":8724.34},"usIn":1573756522511975,"usOut":1573756522512240,"usDiff":265,"testnet":true}
I decided to rewrite it the OOP way, and here's the class I created (the file is named "Call_Deribit"):
import asyncio, websockets, json
class WSClient():
def __init__(self, key=None, secret=None, url=None):
self.api_key = key
self.api_secret = secret
self.msg = {
"jsonrpc": "2.0",
"id": 0
}
if url:
self.host = url
else:
self.host = 'wss://test.deribit.com/ws/api/v2'
async def call_api(self, msg):
async with websockets.connect(self.host) as websocket:
print("Connected to URL:", self.host)
try:
await websocket.send(msg)
while websocket.open:
response = await websocket.recv()
response_json = json.loads(response)
return response_json
except Exception as e:
return e
def request(self, method, params, session=None):
msg = self.msg
msg["id"] += 1
msg["method"] = method
msg["params"] = params
if session != None:
msg["params"]["scope": "session:{}".format(session)]
return asyncio.get_event_loop().run_until_complete(self.call_api(json.dumps(msg)))
def get_order_book(self, instrument):
method = "public/get_order_book"
params = {
"instrument_name": instrument
}
return self.request(method, params)
And here's the main file I'm accessing the class from and where I make all the requests:
import json, asyncio, websockets
from Call_Deribit import WSClient
CL_ID = 'qxv0EeAu'
CL_SECRET = 't24F49ocH1_qFawiKnEyqlWF5D-haABb31O8xCQhySg'
REQ_URL = 'wss://test.deribit.com/ws/api/v2'
method_auth = "public/auth"
params_auth = {
"grant_type": "client_credentials",
"client_id": CL_ID,
"client_secret": CL_SECRET
}
main_client = WSClient(key=CL_ID, secret=CL_SECRET, url=REQ_URL)
auth_response = main_client.request(method_auth, params_auth)
acc_token = auth_response["result"]["access_token"]
method_pos = "private/get_position"
params_pos = {
"access_token": acc_token,
"instrument_name": "BTC-PERPETUAL"
}
position = main_client.request(method_pos, params_pos)
print(position)
The first request for authentication is working this time, and I'm able to extract the access token as well, but the second private/get_position message is, for whatever reason, returning an unauthorized error.
{'jsonrpc': '2.0', 'id': 1, 'error': {'message': 'unauthorized', 'code': 13009}, 'testnet': True, 'usIn': 1573756936534405, 'usOut': 1573756936534629, 'usDiff': 224}
I've spent hours on it, and I seem to be doing exactly the same thing in the OOP version as I did on the original one. My familiarity with OOP and its concepts (such as inheritance) is limited, so I'd like to know what I'm missing here, and why my code isn't working in the OOP version, despite following the same exact workflow as in the original version.
Here's the documentation for the Deribit API: https://docs.deribit.com/v2/?python#json-rpc
Any help would be greatly appreciated.

Adding the scope under params_auth in the main file works:
params_auth = {
"grant_type": "client_credentials",
"client_id": CL_ID,
"client_secret": CL_SECRET,
"scope": "session:test"
}

python asyncio asynchronously fetch data by key from a dict when the key becomes available

Like title told, my use case is like this:
I have one aiohttp server, which accept request from client, when i have the request i generate one unique request id for it, and then i send the {req_id: req_pyaload} dict to some workers (the worker is not in python thus running in another process), when the workers complete the work, i get back the response and put them in a result dict like this: {req_id_1: res_1, req_id_2: res_2}.
Then I want my aiohttp server handler to await on above result dict, so when the specific response become available (by req_id) it can send it back.
I build below example code to try to simulate the process, but got stuck in implementing the coroutine async def fetch_correct_res(req_id) which should asynchronously/unblockly fetch the correct response by req_id.
import random
import asyncio
import shortuuid
n_tests = 1000
idxs = list(range(n_tests))
req_ids = []
for _ in range(n_tests):
req_ids.append(shortuuid.uuid())
res_dict = {}
async def fetch_correct_res(req_id):
pass
async def handler(req):
res = await fetch_correct_res(req)
assert req == res, "the correct res for the req should exactly be the req itself."
print("got correct res for req: {}".format(req))
async def randomly_put_res_to_res_dict():
for _ in range(n_tests):
random_idx = random.choice(idxs)
await asyncio.sleep(random_idx / 1000)
res_dict[req_ids[random_idx]] = req_ids[random_idx]
print("req: {} is back".format(req_ids[random_idx]))
So:
Is it possible to make this solution work? how?
If above solution is not possible, what should be the correct solution for this use case with asyncio?
Many thanks.
The only approach i can think of for now to make this work is: pre-created some asyncio.Queue with pre-assigned id, then for each incoming request assign one queue to it, so the handler just await on this queue, when the response come back i put it into this pre-assigned queue only, after the request fulfilled, i collect back the queue to use it for next incoming request. Not very elegant, but will solve the problem.

See if the below sample implementation fulfils your need
basically you want to respond back to the request(id) with your response(unable to predict the order) in an asynchronous way
So at the time of request handling, populate the dict with {request_id: {'event':<async.Event>, 'result': <result>}} and await on asyncio.Event.wait(), once the response is received, signal the event with asyncio.Event.set() which will release the await and then fetch the response from the dict based on the request id
I modified your code slightly to pre-populate the dict with request id and put the await on asyncio.Event.wait() until the signal comes from the response
import random
import asyncio
import shortuuid
n_tests = 10
idxs = list(range(n_tests))
req_ids = []
for _ in range(n_tests):
req_ids.append(shortuuid.uuid())
res_dict = {}
async def fetch_correct_res(req_id, event):
await event.wait()
res = res_dict[req_id]['result']
return res
async def handler(req, loop):
print("incoming request id: {}".format(req))
event = asyncio.Event()
data = {req :{}}
res_dict.update(data)
res_dict[req]['event']=event
res_dict[req]['result']='pending'
res = await fetch_correct_res(req, event)
assert req == res, "the correct res for the req should exactly be the req itself."
print("got correct res for req: {}".format(req))
async def randomly_put_res_to_res_dict():
random.shuffle(req_ids)
for i in req_ids:
await asyncio.sleep(random.randrange(2,4))
print("req: {} is back".format(i))
if res_dict.get(i) is not None:
event = res_dict[i]['event']
res_dict[i]['result'] = i
event.set()
loop = asyncio.get_event_loop()
tasks = asyncio.gather(handler(req_ids[0], loop),
handler(req_ids[1], loop),
handler(req_ids[2], loop),
handler(req_ids[3], loop),
randomly_put_res_to_res_dict())
loop.run_until_complete(tasks)
loop.close()
sample response from the above code
incoming request id: NDhvBPqMiRbteFD5WqiLFE
incoming request id: fpmk8yC3iQcgHAJBKqe2zh
incoming request id: M7eX7qeVQfWCCBnP4FbRtK
incoming request id: v2hAfcCEhRPUDUjCabk45N
req: VeyvAEX7YGgRZDHqa2UGYc is back
req: M7eX7qeVQfWCCBnP4FbRtK is back
got correct res for req: M7eX7qeVQfWCCBnP4FbRtK
req: pVvYoyAzvK8VYaHfrFA9SB is back
req: soP8NDxeQKYjgeT7pa3wtG is back
req: j3rcg5Lp59pQXuvdjCAyZe is back
req: NDhvBPqMiRbteFD5WqiLFE is back
got correct res for req: NDhvBPqMiRbteFD5WqiLFE
req: v2hAfcCEhRPUDUjCabk45N is back
got correct res for req: v2hAfcCEhRPUDUjCabk45N
req: porzHqMqV8SAuttteHRwNL is back
req: trVVqZrUpsW3tfjQajJfb7 is back
req: fpmk8yC3iQcgHAJBKqe2zh is back
got correct res for req: fpmk8yC3iQcgHAJBKqe2zh

This may work (note: I removed UUID in order to know req id in advance)
import random
import asyncio
n_tests = 1000
idxs = list(range(n_tests))
req_ids = []
for i in range(n_tests):
req_ids.append(i)
res_dict = {}
async def fetch_correct_res(req_id):
while not res_dict.get(req_id):
await asyncio.sleep(0.1)
return req_ids[req_id]
async def handler(req):
print("fetching req: ", req)
res = await fetch_correct_res(req)
assert req == res, "the correct res for the req should exactly be the req itself."
print("got correct res for req: {}".format(req))
async def randomly_put_res_to_res_dict(future):
for i in range(n_tests):
res_dict[req_ids[i]] = req_ids[i]
await asyncio.sleep(0.5)
print("req: {} is back".format(req_ids[i]))
future.set_result("done")
loop = asyncio.get_event_loop()
future = asyncio.Future()
asyncio.ensure_future(randomly_put_res_to_res_dict(future))
loop.run_until_complete(handler(10))
loop.close()
Is it the best solution? according to me No, basically its kind of requesting long running job status, and you should have (REST) api for doing the job submission and knowing job status like:
http POST server:port/job
{some job json paylod}
Response: 200 OK {"req_id": 1}
http GET server:port/job/1
Response: 200 OK {"req_id": 1, "status": "in process"}
http GET server:port/job/1
Response: 200 OK {"req_id": 1, "status": "done", "result":{}}