I am trying to create a data endpoint that streams either the entirety of a file or responds appropriately to range requests. Streaming the whole file seems understandable, but it's not clear to me how to deal with range requests. Particularly, I can't see how aiohttp.MultipartWriter can write to a StreamResponse.
Here's an abstracted form of my code, so far:
from aiohttp.web import Request, StreamResponse
from aiohttp.multipart import MultipartWriter
async def data_handler(req:Request) -> StreamResponse:
is_range_request = "Range" in req.headers
with open("my_big_file", "rb") as f:
if is_range_request:
status_code = 202
content_type = "multipart/bytes"
else:
status_code = 200
content_type = "application/octet-stream"
resp = SteamResponse(status=status_code, headers={"Content-Type": content_type})
resp.enable_chunked_encoding()
resp.enable_compression()
await resp.prepare(req)
if is_range_request:
# _parse_range_header :: str -> List[ByteRange]
# ByteRange = Tuple[int, int] i.e., "from" and "to", inclusive
ranges = _parse_range_header(req.headers["Range"])
mpwriter = MultipartWriter("bytes")
for r in ranges:
range_from, range_to = r
range_size = (range_to - range_from) + 1
range_header = {"Content-Type": "application/octet-stream"}
# FIXME Won't this block?
f.seek(range_from)
mpwriter.append(f.read(range_size), range_header)
# TODO Write to response. How?...
else:
while True:
data = f.read(8192)
if not data:
await resp.drain()
break
resp.write(data)
return resp
This also doesn't return the response until it gets to the end. This doesn't seem correct to me: How does an upstream call know what's going on until the response is returned; or is the asyncio stuff doing this for me automagically?
Related
The below is a result of this question How to sign an OKEx API request? and some of the answers:
import hmac
import base64
import requests
import datetime
import json
from config import KEY, SECRET, PASS, ROOT_URL
def get_time():
now = datetime.datetime.utcnow()
t = now.isoformat("T", "milliseconds")
return t + "Z"
def signature(timestamp, request_type, endpoint, body, secret):
if body != '':
body = json.dumps(body)
message = str(timestamp) + str.upper(request_type) + endpoint + body
print(message)
mac = hmac.new(bytes(secret, encoding='utf-8'), bytes(message, encoding='utf-8'), digestmod='sha256')
d = mac.digest()
return base64.b64encode(d)
def get_header(request_type, endpoint, body):
time = get_time()
header = dict()
header['CONTENT-TYPE'] = 'application/json'
header['OK-ACCESS-KEY'] = KEY
header['OK-ACCESS-SIGN'] = signature(time, request_type, endpoint, body, SECRET)
header['OK-ACCESS-TIMESTAMP'] = str(time)
header['OK-ACCESS-PASSPHRASE'] = PASS
return header
def get(endpoint, body=''):
url = ROOT_URL + endpoint
header = get_header('GET', endpoint, body)
return requests.get(url, headers=header)
def post(endpoint, body=''):
url = ROOT_URL + endpoint
header = get_header('POST', endpoint, body)
return requests.post(url, headers=header)
where KEY, SECRET, PASS are the API key, secret key, and pass phrase respectively; The ROOT_URL is 'https://www.okex.com'.
The Problem
GET requests work absolutely fine, so when I run the following, there are no issues:
ENDPOINT = '/api/v5/account/balance'
BODY = ''
response = get(ENDPOINT)
response.json()
However, when I try to place an order via a POST request, like so:
ENDPOINT = '/api/v5/trade/order'
BODY = {"instId":"BTC-USDT",
"tdMode":"cash",
"side":"buy",
"ordType":"market",
"sz":"1"}
response = post(ENDPOINT, body=BODY)
response.json()
I get the following output, i.e. it won't accept the signature:
{'msg': 'Invalid Sign', 'code': '50113'}
Related Questions
In this one Can't figure out how to send a signed POST request to OKEx an answer was provided, but it does not work for me as I was already using the suggested URL. More or less the same question was asked here Unable to send a post requests OKEX Invalid Signature, but no activity likely due to the format, so I thought I would repost and elaborate.
OKEX Docs
The docs simply specify that The API endpoints of Trade require authentication (https://www.okex.com/docs-v5/en/?python#rest-api-authentication-signature). But they make no reference to there being any difference between the two methods. Away from that, I am including all required parameters in the body of the post request as far as I can see.
I would appreciate any input on this.
Many thanks!
I ran into the same POST problem and figured it out. I used new domain name okex.com. Here is my code.
def set_userinfo(self):
position_path = "/api/v5/account/set-position-mode"
try:
self.get_header("POST", position_path, {"posMode":"net_mode"})
resp = requests.post(url=self.base_url+position_path, headers=self.headers, json={"posMode":"long_short_mode"}).json()
except Exception as e:
log.error("OK set_userinfo error={} type={}".format(f'{e}', f'{type(e)}'))
def get_header(self, request_type, endpoint, body=''):
timestamp = self.get_time()
self.headers["OK-ACCESS-TIMESTAMP"] = timestamp
self.headers["OK-ACCESS-SIGN"] = self.signature(timestamp, request_type, endpoint, body)
def signature(self, timestamp, request_type, endpoint, body):
if body != '':
body = json.dumps(body)
message = str(timestamp) + str.upper(request_type) + endpoint + body
mac = hmac.new(bytes(self.secret_key, encoding='utf-8'), bytes(message, encoding='utf-8'), digestmod='sha256').digest()
return base64.b64encode(mac)
I have fix the same problem.
Both of the 'body' in signature() and in get_header() should be json.
So you should add following code:
if str(body) == '{}' or str(body) == 'None':
body = ''
else:
body = json.dumps(body)
I ran into the same problem and solved it using below code snippet, the idea is from https://stackoverflow.com/a/68115787/20497127, but I modified a little by adding POST functionality
APIKEY = "" # input key
APISECRET = "" #input secret
PASS = "" #input passphrase
BASE_URL = 'https://www.okx.com'
def send_signed_request(http_method, url_path, payload={}):
def get_time():
return dt.datetime.utcnow().isoformat()[:-3]+'Z'
def signature(timestamp, method, request_path, body, secret_key):
if str(body) == '{}' or str(body) == 'None':
body = ''
message = str(timestamp) + str.upper(method) + request_path + str(body)
mac = hmac.new(bytes(secret_key, encoding='utf8'), bytes(message, encoding='utf-8'), digestmod='sha256')
d = mac.digest()
return base64.b64encode(d)
# set request header
def get_header(request='GET', endpoint='', body:dict=dict()):
cur_time = get_time()
header = dict()
header['CONTENT-TYPE'] = 'application/json'
header['OK-ACCESS-KEY'] = APIKEY
header['OK-ACCESS-SIGN'] = signature(cur_time, request, endpoint , body, APISECRET)
header['OK-ACCESS-TIMESTAMP'] = str(cur_time)
header['OK-ACCESS-PASSPHRASE'] = PASS
# demo trading: need to set x-simulated-trading=1, live trading is 0
header['x-simulated-trading'] = '1'
return header
url = BASE_URL + url_path
header = get_header(http_method, url_path, payload)
print(url)
print(header)
if http_method == 'GET':
response = requests.get(url, headers=header)
elif http_method == 'POST':
response = requests.post(url, headers=header, data=payload)
return response.json()
# this will run get requests
res = send_signed_request("GET", "/api/v5/account/balance", payload={})
# this will run post requests
data = {
"instId": "BTC-USDT",
"tdMode": "cross",
"side": "sell",
"ccy":"USDT",
"ordType": "limit",
"px": "100000",
"sz": "0.01"
}
res = send_signed_request("POST", "/api/v5/trade/order", payload=json.dumps(data))
I have csv file from which one column serves as the data that is passed in two of the POST method calls. I have three different URLs which need to be called subsequently in the code. I define the session using aiohttp and loop for the column length(one whose cell values are sent in two of the POST calls). The data obtained from the responses is stored in a list and that is used for a third POST call with different url.
The first two calls seem to execute fine and pretty fast(i printed the responses for them and verified), but when execution comes to the third one, this error shows up:
OSError: [Errno 24] Too many open files
I have tried some solutions which suggest to specify connector=aiohttp.TCPConnector(verify_ssl=False) in aiohttp.ClientSession(connector=aiohttp.TCPConnector(verify_ssl=False))
But this doesn't work. What should be the best way to tackle this? The second URL curently that i use is an HTTP one, localhost url, which will be changed to https cloud based url eventually.
Here is the example code depicting the situation:
import aiohttp
import pandas as pd
import asyncio
import requests
import json
import time
start_time = time.time()
df = pd.read_csv('Sample.csv', header=None, dtype=str)
RequestBodyvalues = df.iloc[:, [0]].values
async def get_FirstUrlresponse(session, url, requestBody, headers):
async with session.post(url, data = json.dumps(requestBody), headers = headers) as resp:
response = await resp.json()
return response
async def get_SecondUrlresponse(session, url, requestBody, headers):
async with session.post(url, data = json.dumps(requestBody), headers = headers) as resp:
response = await resp.json()
return response
async def get_ThirdUrlresponse(session, url, requestBody, headers):
async with session.post(url, data = json.dumps(requestBody), headers = headers) as resp:
response = await resp.json()
return response
async def main():
async with aiohttp.ClientSession() as session:
FirstUrlTasks = []
SecondUrlTasks = []
for reqBody in RequestBodyvalues:
firstUrl = 'https://firstUrl.com/searchByValue'
secondUrl = 'http://secondUrl.com/someEndpoint'
requestBody = {'value': reqBody}
headers = {'Authorization' : 'Bearer Token',
'content-type': 'application/json'}
FirstUrlTasks.append(asyncio.ensure_future(get_FirstUrlresponse(session, firstUrl, requestBody, headers)))
SecondUrlTasks.append(asyncio.ensure_future(get_SecondUrlresponse(session, secondUrl, requestBody, headers)))
firstUrlResponses = await asyncio.gather(*FirstUrlTasks)
secondUrlresponses = await asyncio.gather(*SecondUrlTasks)
valuesForThridUrl = []
for secondUrlresponse in secondUrlresponses:
#Logic to fetch values to pass to Third Url stored in list
ThirdUrlTasks = []
for value in valuesForThridUrl:
ThirdUrl = 'https://thirdUrl.com/someEndpoint'
requestBody = {'reqBody': value}
headers = {'Authorization' : 'Bearer Token',
'content-type': 'application/json'}
ThirdUrlTasks.append(asyncio.ensure_future(get_ThirdUrlresponse(session, ThirdUrl, requestBody, headers)))
thirdUrlresponses = await asyncio.gather(*ThirdUrlTasks)
asyncio.run(main())
See open file limits with this command
ulimit -n
and then increase limit
ulimit -n NUM
I tried to get the ice cast meta data of a mp3 stream with this script:
import requests
url = 'http://stream.jam.fm/jamfm-nmr/mp3-128/konsole/'
try:
response = requests.get(url, headers={'Icy-MetaData': 1}, stream=True)
response.raise_for_status()
except requests.RequestException, e:
print 'Error:', e
else:
headers, stream = response.headers, response.raw
meta_int = headers.get('icy-metaint')
if meta_int is not None:
audio_length = int(meta_int)
while True:
try:
audio_data = stream.read(audio_length)
meta_byte = stream.read(1)
if (meta_byte):
meta_length = ord(meta_byte) * 16
meta_data = stream.read(meta_length)
print meta_data
except KeyboardInterrupt:
break
response.close()
This works but just for the first package. I will never receive an update on the title information when the track changes. My question is: Is this intended behavior and the track info is just send once or did I something wrong? I would like to be able to notice a track change without polling the stream from time to time.
while True:
try:
#new request
response = requests.get(url, headers={'Icy-MetaData': 1}, stream=True)
response.raise_for_status()
headers, stream = response.headers, response.raw
meta_int = headers.get('icy-metaint')
audio_data = stream.read(audio_length)
meta_byte = stream.read(1)
if (meta_byte):
meta_length = ord(meta_byte) * 16
meta_data = stream.read(meta_length)
print (meta_data)
except KeyboardInterrupt:
break
def do_request(url, token, json_data=None,
mode="get", work_around_for_image_custom_list=False):
"""Uploads a file. """
header_collection = {"X-Auth-Token": token}
if json_data is not None:
header_collection['Content-Type'] = 'application/json'
try:
if mode == "delete":
# this looks ugly, but there is absolutely no way to
# get requests to do DELETE when there is a blank JSON
# included
r = requests.delete(url, headers=header_collection, timeout=10)
else:
r = getattr(requests, mode)(url, data=json.dumps(json_data),
headers=header_collection, timeout=10)
if r.status_code == 200:
#This looks ugly also, but has to be for a particular function that calls it
if work_around_for_image_custom_list:
return r
else:
http_info = (json.dumps(r.json(), indent=2), r.status_code)
else:
http_info = (r.text, r.status_code)
return http_info
except requests.exceptions.ConnectionError:
print "Connection Error! Http status Code {}".format(r.status_code)
sys.exit()
except (requests.exceptions.RequestException,
requests.exceptions.HTTPError):
print "Ambiguous Error! Http status Code {}".format(r.status_code)
sys.exit()
Using Python 2.7 and the requests module, I have this function that I call with several other functions to make api calls. However, I have to make an exception for one particular function and return the request object...if work_around_for_image_custom_list is True. This seems seems like a ugly hack/work around and I am wondering how I could re-write it to compensate if work_around_for_image_custom_list. For instance, would it be better to make this a class and have each function create a object to use it? If so, how would I over ride if r.status_code == 200:?
Expanding on the comment I made:
def do_raw_request(url, token, json_data=None, mode="get"):
"""Uploads a file. """
header_collection = {"X-Auth-Token": token}
if json_data is not None:
header_collection['Content-Type'] = 'application/json'
try:
if mode == "delete":
# this looks ugly, but there is absolutely no way to
# get requests to do DELETE when there is a blank JSON
# included
r = requests.delete(url, headers=header_collection, timeout=10)
else:
r = getattr(requests, mode)(url, data=json.dumps(json_data),
headers=header_collection, timeout=10)
if r.status_code == 200:
return r, r.status_code
return r.text, r.status_code
except requests.exceptions.ConnectionError:
print "Connection Error! Http status Code {}".format(r.status_code)
sys.exit()
except (requests.exceptions.RequestException,
requests.exceptions.HTTPError):
print "Ambiguous Error! Http status Code {}".format(r.status_code)
sys.exit()
Then:
def do_request(url, token, json_data=None, mode="get"):
res, code = do_raw_request(url, token, json_data, mode)
if code == 200:
return (json.dumps(r.json(), indent=2), r.status_code)
return res, code
and now you call either do_raw_request or do_request as appropriate.
Note that I changed the return so it always returns a tuple otherwise you would have to start checking types to know whether you have a status text or a response object.
I am trying to upload a file using multipart_encode to realize the MIME process. However, I met the following error AttributeError: multipart_yielder instance has no attribute '__len__'. Below are is my approach, I really appreciate if anyone can give me some suggestions.
url = "https://pi-user-files.s3-external-1.amazonaws.com/"
post_data = {}
#data is a dict
post_data['AWSAccessKeyId']=(data['ticket']['AWSAccessKeyId'])
post_data['success_action_redirect']=(data['ticket']['success_action_redirect'])
post_data['acl']=(data['ticket']['acl'])
post_data['key']=(data['ticket']['key'])
post_data['signature']=(data['ticket']['signature'])
post_data['policy']=(data['ticket']['policy'])
post_data['Content-Type']=(data['ticket']['Content-Type'])
#I would like to upload a text file "new 2"
post_data['file']=open("new 2.txt", "rb")
datagen, headers = multipart_encode(post_data)
request2 = urllib2.Request(url, datagen, headers)
result = urllib2.urlopen(request2)
If you want to send a file you should wrap other parameters with a MultipartParam object, example code for creating a send file request:
from poster.encode import multipart_encode, MultipartParam
import urllib2
def postFileRequest(url, paramName, fileObj, additionalHeaders={}, additionalParams={}):
items = []
#wrap post parameters
for name, value in additionalParams.items():
items.append(MultipartParam(name, value))
#add file
items.append(MultipartParam.from_file(paramName, fileObj))
datagen, headers = multipart_encode(items)
#add headers
for item, value in additionalHeaders.iteritems():
headers[item] = value
return urllib2.Request(url, datagen, headers)
Also I think you should execute register_openers() once at the beginning. Some details you can find in docs
The problem is that in httplib.py, the generator is not detected as such and is treated instead like a string that holds the full data to be sent (and therefore it tries to find its length):
if hasattr(data,'read') and not isinstance(data, array): # generator
if self.debuglevel > 0: print "sendIng a read()able"
....
A solution is to make the generator act like a read()able:
class GeneratorToReadable():
def __init__(self, datagen):
self.generator = datagen
self._end = False
self.data = ''
def read(self, n_bytes):
while not self._end and len(self.data) < n_bytes:
try:
next_chunk = self.generator.next()
if next_chunk:
self.data += next_chunk
else:
self._end = True
except StopIteration:
self._end = True
result = self.data[0:n_bytes]
self.data = self.data[n_bytes:]
return result
and use like so:
datagen, headers = multipart_encode(post_data)
readable = GeneratorToReadable(datagen)
req = urllib2.Request(url, readable, headers)
result = urllib2.urlopen(req)