I am trying to create a data endpoint that streams either the entirety of a file or responds appropriately to range requests. Streaming the whole file seems understandable, but it's not clear to me how to deal with range requests. Particularly, I can't see how aiohttp.MultipartWriter can write to a StreamResponse.
Here's an abstracted form of my code, so far:
from aiohttp.web import Request, StreamResponse
from aiohttp.multipart import MultipartWriter
async def data_handler(req:Request) -> StreamResponse:
is_range_request = "Range" in req.headers
with open("my_big_file", "rb") as f:
if is_range_request:
status_code = 202
content_type = "multipart/bytes"
else:
status_code = 200
content_type = "application/octet-stream"
resp = SteamResponse(status=status_code, headers={"Content-Type": content_type})
resp.enable_chunked_encoding()
resp.enable_compression()
await resp.prepare(req)
if is_range_request:
# _parse_range_header :: str -> List[ByteRange]
# ByteRange = Tuple[int, int] i.e., "from" and "to", inclusive
ranges = _parse_range_header(req.headers["Range"])
mpwriter = MultipartWriter("bytes")
for r in ranges:
range_from, range_to = r
range_size = (range_to - range_from) + 1
range_header = {"Content-Type": "application/octet-stream"}
# FIXME Won't this block?
f.seek(range_from)
mpwriter.append(f.read(range_size), range_header)
# TODO Write to response. How?...
else:
while True:
data = f.read(8192)
if not data:
await resp.drain()
break
resp.write(data)
return resp
This also doesn't return the response until it gets to the end. This doesn't seem correct to me: How does an upstream call know what's going on until the response is returned; or is the asyncio stuff doing this for me automagically?
I would like to modify this script to use offline files, if I download the files from url works, but if the same file as I withdraw from hard drives, does not open, someone helps me to understand why and how to do, thank you.
def INDEX():
TVLIST('https://www.*********/playlist/*******/test.m3u')
def TVLIST(url):
try:
m3u = getHtml(url)
parsem3u(m3u)
except:
addDir('Nothing found', '', '', '', Folder=False)
xbmcplugin.endOfDirectory(int(sys.argv[1]))
urlopen = urllib2.urlopen
Request = urllib2.Request
def getHtml(url, referer=None, hdr=None, data=None):
if not hdr:
req = Request(url, data, headers)
else:
req = Request(url, data, hdr)
if referer:
req.add_header('Referer', referer)
if data:
req.add_header('Content-Length', len(data))
response = urlopen(req)
if response.info().get('Content-Encoding') == 'gzip':
buf = StringIO( response.read())
f = gzip.GzipFile(fileobj=buf)
data = f.read()
f.close()
else:
data = response.read()
response.close()
return data
def parsem3u(html, sitechk=True):
match = re.compile('#.+,(.+?)\n(.+?)\n').findall(html)
txtfilter = txtfilter = GETFILTER()
txtfilter = txtfilter.split(',') if txtfilter else []
txtfilter = [f.lower().strip() for f in txtfilter]
i = 0
count = 0
for name, url in match:
status = ""
url = url.replace('\r','')
if not txtfilter or any(f in name.lower() for f in txtfilter):
if sitechk:
if i < 5:
try:
siteup = urllib.urlopen(url).getcode()
status = " [COLOR red]offline[/COLOR]" if siteup != 200 else " [COLOR green]online[/COLOR]"
except: status = " [COLOR red]offline[/COLOR]"
i += 1
addPlayLink(name+status, url, 3, uiptvicon)
count += 1
return count
I thought, was enough to put the local path
def INDEX():
TVLIST(r'c:\Desktop\IPTVLIST\M3U\playlist\test.m3u')
who explains why it does not work and how can I do? Thank you
As suggested by #languitar in the comments you would have file:// which of course it should work for windows, but moving to a platform like android, you have different file system there, you don't have C drive. So make sure you got an alternative location on the android.
I've been trying for hours, and I just don't know what I'm doing wrongly. It's just for planning/research (not performant) -- playing around with some code from github -- but I need to see it functional.
RPC_USER = username
RPC_PASS = pasword
rpc_id = ID HERE
jsonrpc = "2.0"
payload = {"jsonrpc": jsonrpc, "id": rpc_id, "method": method, "params": params}
authstr = base64.encodestring(bytes('%s:%s' % (RPC_USER, RPC_PASS), 'utf-8')).strip()
request_headers = {"Authorization": "Basic %s" % authstr, 'content-type': 'application/json'}
try:
response = requests.get(RPC_URL, headers = request_headers, data = json.dumps(payload)).json()
print(response['result'])
except Exception as e: print(str(e))
if response['id'] != rpc_id:
raise ValueError("invalid response id!")
I get an error as follows:
Here's the whole traceback:
Expecting value: line 1 column 1 (char 0) # prints the Exception
Traceback (most recent call last):
File "miner_2017.py", line 411, in <module>
solo_miner(bin2hex("------coinbase message here -----"), "-----bitcoin address here-----")
File "miner_2017.py", line 401, in solo_miner
mined_block, hps = block_mine(rpc_getblocktemplate(), coinbase_message, 0, address, timeout=60)
File "miner_2017.py", line 63, in rpc_getblocktemplate
try: return rpc("getblocktemplate", [{}])
File "miner_2017.py", line 52, in rpc
if response['id'] != rpc_id:
UnboundLocalError: local variable 'response' referenced before assignment
Which after doing some looking seems to be a problem with decoding the json object from a bytes object rather than a string object. I don't know how to fix this. It seems the "response" variable assignment was unsuccessful due to the json problem. How can I get the json object in string form from the request?
Would somebody help me out? Thanks
#!/usr/bin/env python
import getpass
import json
import requests
def instruct_wallet(method, params):
url = "http://127.0.0.1:8332/"
payload = json.dumps({"method": method, "params": params})
headers = {'content-type': "application/json", 'cache-control': "no-cache"}
try:
response = requests.request("POST", url, data=payload, headers=headers, auth=(rpc_user, rpc_password))
return json.loads(response.text)
except requests.exceptions.RequestException as e:
print e
except:
print 'No response from Wallet, check Bitcoin is running on this machine'
rpc_user='foo'
rpc_password='bar'
passphrase = getpass.getpass('Enter your wallet passphrase: ')
timeout = raw_input('Unlock for how many seconds: ')
answer = instruct_wallet('walletpassphrase', [passphrase, timeout])
if answer['error'] != None:
print answer['error']
else:
print answer['result']
I'm using something similar for Altcoins
import decimal
import itertools
import json
import requests
id_counter = itertools.count()
class BTCJsonRPC(object):
def __init__(self, url, user, passwd, log, method=None, timeout=30):
self.url = url
self._user = user
self._passwd = passwd
self._method_name = method
self._timeout = timeout
self._log = log
def __getattr__(self, method_name):
return BTCJsonRPC(self.url, self._user, self._passwd, self._log, method_name, timeout=self._timeout)
def __call__(self, *args):
# rpc json call
playload = json.dumps({'jsonrpc': '2.0', 'id': next(id_counter), "method": self._method_name, "params": args})
headers = {'Content-type': 'application/json'}
resp = None
try:
resp = requests.post(self.url, headers=headers, data=playload, timeout=self._timeout,
auth=(self._user, self._passwd))
resp = resp.json(parse_float=decimal.Decimal)
except Exception as e:
error_msg = resp.text if resp is not None else e
msg = u"{} {}:[{}] \n {}".format('post', self._method_name, args, error_msg)
self._log.error(msg)
return
if resp.get('error') is not None:
e = resp['error']
self._log.error('{}:[{}]\n {}:{}'.format(self._method_name, args, e['code'], e['message']))
return None
elif 'result' not in resp:
self._log.error('[{}]:[{}]\n MISSING JSON-RPC RESULT'.format(self._method_name, args, ))
return None
return resp['result']
I'm pretty sure you just need to change from using a GET to a POST, i.e.:
change
response = requests.get(RPC_URL, headers = request_headers, data = json.dumps(payload)).json()
to
response = requests.post(RPC_URL, headers=request_headers, data=json.dumps(payload)).json()
In fact, when I tried this with GET (without dumping the response to json), I got a 405 response. You should always take a look at your response object before doing further debugging with it.
I am uploading a large file using the Python requests package, and I can't find any way to give data back about the progress of the upload. I have seen a number of progress meters for downloading a file, but these will not work for a file upload.
The ideal solution would be some sort of callback method such as:
def progress(percent):
print percent
r = requests.post(URL, files={'f':hugeFileHandle}, callback=progress)
Thanks in advance for your help :)
requests doesn't support upload streaming e.g.:
import os
import sys
import requests # pip install requests
class upload_in_chunks(object):
def __init__(self, filename, chunksize=1 << 13):
self.filename = filename
self.chunksize = chunksize
self.totalsize = os.path.getsize(filename)
self.readsofar = 0
def __iter__(self):
with open(self.filename, 'rb') as file:
while True:
data = file.read(self.chunksize)
if not data:
sys.stderr.write("\n")
break
self.readsofar += len(data)
percent = self.readsofar * 1e2 / self.totalsize
sys.stderr.write("\r{percent:3.0f}%".format(percent=percent))
yield data
def __len__(self):
return self.totalsize
# XXX fails
r = requests.post("http://httpbin.org/post",
data=upload_in_chunks(__file__, chunksize=10))
btw, if you don't need to report progress; you could use memory-mapped file to upload large file.
To workaround it, you could create a file adaptor similar to the one from
urllib2 POST progress monitoring:
class IterableToFileAdapter(object):
def __init__(self, iterable):
self.iterator = iter(iterable)
self.length = len(iterable)
def read(self, size=-1): # TBD: add buffer for `len(data) > size` case
return next(self.iterator, b'')
def __len__(self):
return self.length
Example
it = upload_in_chunks(__file__, 10)
r = requests.post("http://httpbin.org/post", data=IterableToFileAdapter(it))
# pretty print
import json
json.dump(r.json, sys.stdout, indent=4, ensure_ascii=False)
I recommend to use a tool package named requests-toolbelt, which make monitoring upload bytes very easy, like
from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
import requests
def my_callback(monitor):
# Your callback function
print monitor.bytes_read
e = MultipartEncoder(
fields={'field0': 'value', 'field1': 'value',
'field2': ('filename', open('file.py', 'rb'), 'text/plain')}
)
m = MultipartEncoderMonitor(e, my_callback)
r = requests.post('http://httpbin.org/post', data=m,
headers={'Content-Type': m.content_type})
And you may want to read this to show a progress bar.
I got it working with the code from here: Simple file upload progressbar in PyQt.
I changed it a bit, to use BytesIO instead of StringIO.
class CancelledError(Exception):
def __init__(self, msg):
self.msg = msg
Exception.__init__(self, msg)
def __str__(self):
return self.msg
__repr__ = __str__
class BufferReader(BytesIO):
def __init__(self, buf=b'',
callback=None,
cb_args=(),
cb_kwargs={}):
self._callback = callback
self._cb_args = cb_args
self._cb_kwargs = cb_kwargs
self._progress = 0
self._len = len(buf)
BytesIO.__init__(self, buf)
def __len__(self):
return self._len
def read(self, n=-1):
chunk = BytesIO.read(self, n)
self._progress += int(len(chunk))
self._cb_kwargs.update({
'size' : self._len,
'progress': self._progress
})
if self._callback:
try:
self._callback(*self._cb_args, **self._cb_kwargs)
except: # catches exception from the callback
raise CancelledError('The upload was cancelled.')
return chunk
def progress(size=None, progress=None):
print("{0} / {1}".format(size, progress))
files = {"upfile": ("file.bin", open("file.bin", 'rb').read())}
(data, ctype) = requests.packages.urllib3.filepost.encode_multipart_formdata(files)
headers = {
"Content-Type": ctype
}
body = BufferReader(data, progress)
requests.post(url, data=body, headers=headers)
The trick is, to generate data and header from the files list manually, using encode_multipart_formdata() from urllib3
I know this is an old question, but I couldn't find an easy answer anywhere else, so hopefully this will help somebody else:
import requests
import tqdm
with open(file_name, 'rb') as f:
r = requests.post(url, data=tqdm(f.readlines()))
This solution uses requests_toolbelt and tqdm both well maintained and popular libraries.
from pathlib import Path
from tqdm import tqdm
import requests
from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
def upload_file(upload_url, fields, filepath):
path = Path(filepath)
total_size = path.stat().st_size
filename = path.name
with tqdm(
desc=filename,
total=total_size,
unit="B",
unit_scale=True,
unit_divisor=1024,
) as bar:
with open(filepath, "rb") as f:
fields["file"] = ("filename", f)
e = MultipartEncoder(fields=fields)
m = MultipartEncoderMonitor(
e, lambda monitor: bar.update(monitor.bytes_read - bar.n)
)
headers = {"Content-Type": m.content_type}
requests.post(upload_url, data=m, headers=headers)
Example usage
upload_url = 'https://uploadurl'
fields = {
"field1": value1,
"field2": value2
}
filepath = '97a6fce8_owners_2018_Van Zandt.csv'
upload_file(upload_url, fields, filepath)
Usually you would build a streaming datasource (a generator) that reads the file chunked and reports its progress on the way (see kennethreitz/requests#663. This does not work with requests file-api, because requests doesn’t support streaming uploads (see kennethreitz/requests#295) – a file to upload needs to be complete in memory before it starts getting processed.
but requests can stream content from a generator as J.F. Sebastian has proven before, but this generator needs to generate the complete datastream including the multipart encoding and boundaries. This is where poster comes to play.
poster is originally written to be used with pythons urllib2 and supports streaming generation of multipart requests, providing progress indication as it goes along. Posters Homepage provides examples of using it together with urllib2 but you really don’t want to use urllib2. Check out this example-code on how to to HTTP Basic Authentication with urllib2. Horrrrrrrrible.
So we really want to use poster together with requests to do file uploads with tracked progress. And here is how:
# load requests-module, a streamlined http-client lib
import requests
# load posters encode-function
from poster.encode import multipart_encode
# an adapter which makes the multipart-generator issued by poster accessable to requests
# based upon code from http://stackoverflow.com/a/13911048/1659732
class IterableToFileAdapter(object):
def __init__(self, iterable):
self.iterator = iter(iterable)
self.length = iterable.total
def read(self, size=-1):
return next(self.iterator, b'')
def __len__(self):
return self.length
# define a helper function simulating the interface of posters multipart_encode()-function
# but wrapping its generator with the file-like adapter
def multipart_encode_for_requests(params, boundary=None, cb=None):
datagen, headers = multipart_encode(params, boundary, cb)
return IterableToFileAdapter(datagen), headers
# this is your progress callback
def progress(param, current, total):
if not param:
return
# check out http://tcd.netinf.eu/doc/classnilib_1_1encode_1_1MultipartParam.html
# for a complete list of the properties param provides to you
print "{0} ({1}) - {2:d}/{3:d} - {4:.2f}%".format(param.name, param.filename, current, total, float(current)/float(total)*100)
# generate headers and gata-generator an a requests-compatible format
# and provide our progress-callback
datagen, headers = multipart_encode_for_requests({
"input_file": open('recordings/really-large.mp4', "rb"),
"another_input_file": open('recordings/even-larger.mp4', "rb"),
"field": "value",
"another_field": "another_value",
}, cb=progress)
# use the requests-lib to issue a post-request with out data attached
r = requests.post(
'https://httpbin.org/post',
auth=('user', 'password'),
data=datagen,
headers=headers
)
# show response-code and -body
print r, r.text
My upload server doesn't support Chunk-Encoded so I came up with this solution. It basically just a wrapper around python IOBase and allow tqdm.wrapattr to work seamless.
import io
import requests
from typing import Union
from tqdm import tqdm
from tqdm.utils import CallbackIOWrapper
class UploadChunksIterator(Iterable):
"""
This is an interface between python requests and tqdm.
Make tqdm to be accessed just like IOBase for requests lib.
"""
def __init__(
self, file: Union[io.BufferedReader, CallbackIOWrapper], total_size: int, chunk_size: int = 16 * 1024
): # 16MiB
self.file = file
self.chunk_size = chunk_size
self.total_size = total_size
def __iter__(self):
return self
def __next__(self):
data = self.file.read(self.chunk_size)
if not data:
raise StopIteration
return data
# we dont retrive len from io.BufferedReader because CallbackIOWrapper only has read() method.
def __len__(self):
return self.total_size
fp = "data/mydata.mp4"
s3url = "example.com"
_quiet = False
with open(fp, "rb") as f:
total_size = os.fstat(f.fileno()).st_size
if not _quiet:
f = tqdm.wrapattr(f, "read", desc=hv, miniters=1, total=total_size, ascii=True)
with f as f_iter:
res = requests.put(
url=s3url,
data=UploadChunksIterator(f_iter, total_size=total_size),
)
res.raise_for_status()
Making #jfs' answer better in terms of an informative progress bar.
import math
import os
import requests
import sys
class ProgressUpload:
def __init__(self, filename, chunk_size=1250):
self.filename = filename
self.chunk_size = chunk_size
self.file_size = os.path.getsize(filename)
self.size_read = 0
self.divisor = min(math.floor(math.log(self.file_size, 1000)) * 3, 9) # cap unit at a GB
self.unit = {0: 'B', 3: 'KB', 6: 'MB', 9: 'GB'}[self.divisor]
self.divisor = 10 ** self.divisor
def __iter__(self):
progress_str = f'0 / {self.file_size / self.divisor:.2f} {self.unit} (0 %)'
sys.stderr.write(f'\rUploading {dist_file}: {progress_str}')
with open(self.filename, 'rb') as f:
for chunk in iter(lambda: f.read(self.chunk_size), b''):
self.size_read += len(chunk)
yield chunk
sys.stderr.write('\b' * len(progress_str))
percentage = self.size_read / self.file_size * 100
completed_str = f'{self.size_read / self.divisor:.2f}'
to_complete_str = f'{self.file_size / self.divisor:.2f} {self.unit}'
progress_str = f'{completed_str} / {to_complete_str} ({percentage:.2f} %)'
sys.stderr.write(progress_str)
sys.stderr.write('\n')
def __len__(self):
return self.file_size
# sample usage
requests.post(upload_url, data=ProgressUpload('file_path'))
The key is the __len__ method. Without it, I was getting connection closed errors. That's the only reason you can't just use tqdm + iter to get a simple progress bar.
My python code that works great. Credit : twine
import sys
import tqdm
import requests
import requests_toolbelt
class ProgressBar(tqdm.tqdm):
def update_to(self, n: int) -> None:
self.update(n - self.n)
with open("test.zip", "rb") as fp:
data_to_send = []
session = requests.session()
data_to_send.append(
("files", ("test.zip", fp))
)
encoder = requests_toolbelt.MultipartEncoder(data_to_send)
with ProgressBar(
total=encoder.len,
unit="B",
unit_scale=True,
unit_divisor=1024,
miniters=1,
file=sys.stdout,
) as bar:
monitor = requests_toolbelt.MultipartEncoderMonitor(
encoder, lambda monitor: bar.update_to(monitor.bytes_read)
)
r = session.post(
'http://httpbin.org/post',
data=monitor,
headers={"Content-Type": monitor.content_type},
)
print(r.text)
The code below returns none. How can I fix it? I'm using Python 2.6.
import urllib
URL = "http://download.finance.yahoo.com/d/quotes.csv?s=%s&f=sl1t1v&e=.csv"
symbols = ('GGP', 'JPM', 'AIG', 'AMZN','GGP', 'JPM', 'AIG', 'AMZN')
#symbols = ('GGP')
def fetch_quote(symbols):
url = URL % '+'.join(symbols)
fp = urllib.urlopen(url)
try:
data = fp.read()
finally:
fp.close()
def main():
data_fp = fetch_quote(symbols)
# print data_fp
if __name__ =='__main__':
main()
You have to explicitly return the data from fetch_quote function. Something like this:
def fetch_quote(symbols):
url = URL % '+'.join(symbols)
fp = urllib.urlopen(url)
try:
data = fp.read()
finally:
fp.close()
return data # <======== Return
In the absence of an explicit return statement Python returns None which is what you are seeing.
Your method doesn't explicitly return anything, so it returns None