I would like to POST multipart/form-data encoded data.
I have found an external module that does it: http://atlee.ca/software/poster/index.html
however I would rather avoid this dependency. Is there a way to do this using the standard libraries?
thanks
The standard library does not currently support that. There is cookbook recipe that includes a fairly short piece of code that you just may want to copy, though, along with long discussions of alternatives.
It's an old thread but still a popular one, so here is my contribution using only standard modules.
The idea is the same than here but support Python 2.x and Python 3.x.
It also has a body generator to avoid unnecessarily memory usage.
import codecs
import mimetypes
import sys
import uuid
try:
import io
except ImportError:
pass # io is requiered in python3 but not available in python2
class MultipartFormdataEncoder(object):
def __init__(self):
self.boundary = uuid.uuid4().hex
self.content_type = 'multipart/form-data; boundary={}'.format(self.boundary)
#classmethod
def u(cls, s):
if sys.hexversion < 0x03000000 and isinstance(s, str):
s = s.decode('utf-8')
if sys.hexversion >= 0x03000000 and isinstance(s, bytes):
s = s.decode('utf-8')
return s
def iter(self, fields, files):
"""
fields is a sequence of (name, value) elements for regular form fields.
files is a sequence of (name, filename, file-type) elements for data to be uploaded as files
Yield body's chunk as bytes
"""
encoder = codecs.getencoder('utf-8')
for (key, value) in fields:
key = self.u(key)
yield encoder('--{}\r\n'.format(self.boundary))
yield encoder(self.u('Content-Disposition: form-data; name="{}"\r\n').format(key))
yield encoder('\r\n')
if isinstance(value, int) or isinstance(value, float):
value = str(value)
yield encoder(self.u(value))
yield encoder('\r\n')
for (key, filename, fd) in files:
key = self.u(key)
filename = self.u(filename)
yield encoder('--{}\r\n'.format(self.boundary))
yield encoder(self.u('Content-Disposition: form-data; name="{}"; filename="{}"\r\n').format(key, filename))
yield encoder('Content-Type: {}\r\n'.format(mimetypes.guess_type(filename)[0] or 'application/octet-stream'))
yield encoder('\r\n')
with fd:
buff = fd.read()
yield (buff, len(buff))
yield encoder('\r\n')
yield encoder('--{}--\r\n'.format(self.boundary))
def encode(self, fields, files):
body = io.BytesIO()
for chunk, chunk_len in self.iter(fields, files):
body.write(chunk)
return self.content_type, body.getvalue()
Demo
# some utf8 key/value pairs
fields = [('প্রায়', 42), ('bar', b'23'), ('foo', 'ން:')]
files = [('myfile', 'image.jpg', open('image.jpg', 'rb'))]
# iterate and write chunk in a socket
content_type, body = MultipartFormdataEncoder().encode(fields, files)
You can't do this with the stdlib quickly. Howevewr, see the MultiPartForm class in this PyMOTW. You can probably use or modify that to accomplish whatever you need:
PyMOTW: urllib2 - Library for opening URLs.
Related
I have a nested python dictionary that is serialized into a json string, that I am further converting to a compressed Gzip file and base64 encoding it. However, once I convert it back to the JSON string, it adds \\ to the string, which isn't in the original JSON string before conversion. This happens at each of the nested dictionary levels. These are the functions:
import json
import io
import gzip
import base64
import zlib
class numpy_encoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, np.integer):
return int(obj)
elif isinstance(obj, np.floating):
return float(obj)
elif isinstance(obj, np.ndarray):
return obj.tolist()
else:
return super(numpy_encoder, self).default(obj)
def dict_json_dump(dictionary):
dumped = json.dumps(dictionary, cls = numpy_encoder, separators=(",", ":"))
return dumped
def gzip_json_encoder(json_string):
stream = io.BytesIO()
with gzip.open(filename=stream, mode='wt') as zipfile:
json.dump(json_string, zipfile)
return stream
def base64_encoder(gzip_string):
return base64.b64encode(gzip_string.getvalue())
We can use the functions as follows:
json_dict = pe.dict_json_dump(test_dictionary)
gzip_json = pe.gzip_json_encoder(json_dict)
base64_gzip = pe.base64_encoder(gzip_json)
When I check the base64_gzip with the following function:
json_str = zlib.decompress(base64.b64decode(base64_gzip), 16 + zlib.MAX_WBITS)
I get the JSON string back in a format like this(truncated):
b'"{\\"trainingResults\\":{\\"confusionMatrix\\":{\\"tn\\":2,\\"fn\\":1,\\"tp\\":1,\\"fp\\":1},\\"auc\\":{\\"score\\":0.5,\\"tpr\\":[0.0,0.5,0.5,1.0],\\"fpr\\":[0.0,0.333,0.667,1.0]},\\"f1\\"
This isn't the full string, but the contents of the string itself is accurate. What I'm not sure about is why the back slashes are showing up when I convert it back. Anyone have any suggestions? I tried utf-8 encoding on my JSON as well, with no luck. Any help is appreciated!
You're doing JSON encoding twice: Once in dict_json_dump() and again in gzip_json_encoder(). Since json_string is already encoded, you don't need to call json.dump() in gzip_json_encoder().
def gzip_json_encoder(json_string):
stream = io.BytesIO()
with gzip.open(filename=stream, mode='wt') as zipfile:
zipfile.write(json_string)
return stream
I'm new to python so I am building a simple program to parse YAML to JSON and JSON to YAML.
The yaml2json converts YAML to JSON on a single line, but a JSON validator says it is correct.
This is my code so far:
def parseyaml(inFileType, outFileType):
infile = input('Please enter a {} filename to parse: '.format(inFileType))
outfile = input('Please enter a {} filename to output: '.format(outFileType))
with open(infile, 'r') as stream:
try:
datamap = yaml.safe_load(stream)
with open(outfile, 'w') as output:
json.dump(datamap, output)
except yaml.YAMLError as exc:
print(exc)
print('Your file has been parsed.\n\n')
def parsejson(inFileType, outFileType):
infile = input('Please enter a {} filename to parse: '.format(inFileType))
outfile = input('Please enter a {} filename to output: '.format(outFileType))
with open(infile, 'r') as stream:
try:
datamap = json.load(stream)
with open(outfile, 'w') as output:
yaml.dump(datamap, output)
except yaml.YAMLError as exc:
print(exc)
print('Your file has been parsed.\n\n')
An example of the original YAML vs. the new YAML
Original:
inputs:
webTierCpu:
type: integer
minimum: 2
default: 2
maximum: 5
title: Web Server CPU Count
description: The number of CPUs for the Web nodes
New:
inputs:
dbTierCpu: {default: 2, description: The number of CPUs for the DB node, maximum: 5,
minimum: 2, title: DB Server CPU Count, type: integer}
It doesn't look like its decoding all of the JSON so I'm not sure where I should go next...
Your file is losing its formatting because the original dump routine
by default writes all leaf nodes in YAML flow-style, whereas your input is block style
all the way.
You are also losing the order of the keys, which is first because the JSON parser
uses dict, and second because dump sorts the output.
If you look at your intermediate JSON you already see that the key order is
gone at that point. To preserve that, use the new API to load your YAML
and have a special JSON encoder as a replacement for dump that can
handle the subclasses of Mapping in which the YAML is loaded similar to
this example
from the standard Python doc.
Assuming your YAML is stored in input.yaml:
import sys
import json
from collections.abc import Mapping, Sequence
from collections import OrderedDict
import ruamel.yaml
# if you instantiate a YAML instance as yaml, you have to explicitly import the error
from ruamel.yaml.error import YAMLError
yaml = ruamel.yaml.YAML() # this uses the new API
# if you have standard indentation, no need to use the following
yaml.indent(sequence=4, offset=2)
input_file = 'input.yaml'
intermediate_file = 'intermediate.json'
output_file = 'output.yaml'
class OrderlyJSONEncoder(json.JSONEncoder):
def default(self, o):
if isinstance(o, Mapping):
return OrderedDict(o)
elif isinstance(o, Sequence):
return list(o)
return json.JSONEncoder.default(self, o)
def yaml_2_json(in_file, out_file):
with open(in_file, 'r') as stream:
try:
datamap = yaml.load(stream)
with open(out_file, 'w') as output:
output.write(OrderlyJSONEncoder(indent=2).encode(datamap))
except YAMLError as exc:
print(exc)
return False
return True
yaml_2_json(input_file, intermediate_file)
with open(intermediate_file) as fp:
sys.stdout.write(fp.read())
which gives:
{
"inputs": {
"webTierCpu": {
"type": "integer",
"minimum": 2,
"default": 2,
"maximum": 5,
"title": "Web Server CPU Count",
"description": "The number of CPUs for the Web nodes"
}
}
}
You see that your JSON has the appropriate key order, which we also
need to preserve on loading. You can do that without subclassing
anything, by specifying the loading of JSON objects into the subclass of
Mapping, that the YAML parser is using internally, by providingobject_pairs_hook.
from ruamel.yaml.comments import CommentedMap
def json_2_yaml(in_file, out_file):
with open(in_file, 'r') as stream:
try:
datamap = json.load(stream, object_pairs_hook=CommentedMap)
# if you need to "restore" literal style scalars, etc.
# walk_tree(datamap)
with open(out_file, 'w') as output:
yaml.dump(datamap, output)
except yaml.YAMLError as exc:
print(exc)
return False
return True
json_2_yaml(intermediate_file, output_file)
with open(output_file) as fp:
sys.stdout.write(fp.read())
Which outputs:
inputs:
webTierCpu:
type: integer
minimum: 2
default: 2
maximum: 5
title: Web Server CPU Count
description: The number of CPUs for the Web nodes
And I hope that that is similar enough to your original input to be acceptable.
Notes:
When using the new API I tend to use yaml as the name of the
instance of ruamel.yaml.YAML(), instead of from ruamel import
yaml. That however masks the use of yaml.YAMLError because the
error class is not an attribute of YAML()
If you are developing this kind of stuff, I can recommend removing
at least the user input from the actual functionality. It should be
trivial to write your parseyaml and parsejson to call yaml_2_json resp.
json_2_yaml.
Any comments in your original YAML file will be lost, although
ruamel.yaml can load them. JSON originally did allow comments, but it is
not in the specification and no parsers that I know can output comments.
Since your real file has literal block scalars you have to use some magic to get those back.
Include the following functions that walk a tree, recursing into dict values and list elements and converting any line with an embedded newline to a type that gets output to YAML as a literal blocks style scalar in place (hence no return value):
from ruamel.yaml.scalarstring import PreservedScalarString, SingleQuotedScalarString
from ruamel.yaml.compat import string_types, MutableMapping, MutableSequence
def preserve_literal(s):
return PreservedScalarString(s.replace('\r\n', '\n').replace('\r', '\n'))
def walk_tree(base):
if isinstance(base, MutableMapping):
for k in base:
v = base[k] # type: Text
if isinstance(v, string_types):
if '\n' in v:
base[k] = preserve_literal(v)
elif '${' in v or ':' in v:
base[k] = SingleQuotedScalarString(v)
else:
walk_tree(v)
elif isinstance(base, MutableSequence):
for idx, elem in enumerate(base):
if isinstance(elem, string_types):
if '\n' in elem:
base[idx] = preserve_literal(elem)
elif '${' in elem or ':' in elem:
base[idx] = SingleQuotedScalarString(elem)
else:
walk_tree(elem)
And then do
walk_tree(datamap)
after you load the data from JSON.
With all of the above you should have only one line that differs in your Wordpress.yaml file.
function yaml_validate {
python -c 'import sys, yaml, json; yaml.safe_load(sys.stdin.read())'
}
function yaml2json {
python -c 'import sys, yaml, json; print(json.dumps(yaml.safe_load(sys.stdin.read())))'
}
function yaml2json_pretty {
python -c 'import sys, yaml, json; print(json.dumps(yaml.safe_load(sys.stdin.read()), indent=2, sort_keys=False))'
}
function json_validate {
python -c 'import sys, yaml, json; json.loads(sys.stdin.read())'
}
function json2yaml {
python -c 'import sys, yaml, json; print(yaml.dump(json.loads(sys.stdin.read())))'
}
More useful Bash tricks at http://github.com/frgomes/bash-scripts
I've loaded an xml file (Rhythmbox's database file) into Python 3 via the ElementTree parser. After modifying the tree and writing it to disk (ElementTree.write()) using the ascii encoding all of the ASCII hex characters that are in hex code point are converted to ASCII decimal code point. For example here is a diff containing the copyright symbol:
< <copyright>© WNYC</copyright>
---
> <copyright>© WNYC</copyright>
Is there any way to tell Python/ElementTree not to do this? I'd like all the hex codes to stay in hex code point.
I found a solution. First I created a new codec error handler and then monkey patched ElementTree._get_writer() to use the new error handler. Looks like:
from xml.etree import ElementTree
import io
import contextlib
import codecs
def lower_first(s):
return s[:1].lower() + s[1:] if s else ''
def html_replace(exc):
if isinstance(exc, (UnicodeEncodeError, UnicodeTranslateError)):
s = []
for c in exc.object[exc.start:exc.end]:
s.append('&#%s;' % lower_first(hex(ord(c))[1:].upper()))
return ''.join(s), exc.end
else:
raise TypeError("can't handle %s" % exc.__name__)
codecs.register_error('html_replace', html_replace)
# monkey patch this python function to prevent it from using xmlcharrefreplace
#contextlib.contextmanager
def _get_writer(file_or_filename, encoding):
# returns text write method and release all resources after using
try:
write = file_or_filename.write
except AttributeError:
# file_or_filename is a file name
if encoding == "unicode":
file = open(file_or_filename, "w")
else:
file = open(file_or_filename, "w", encoding=encoding,
errors="html_replace")
with file:
yield file.write
else:
# file_or_filename is a file-like object
# encoding determines if it is a text or binary writer
if encoding == "unicode":
# use a text writer as is
yield write
else:
# wrap a binary writer with TextIOWrapper
with contextlib.ExitStack() as stack:
if isinstance(file_or_filename, io.BufferedIOBase):
file = file_or_filename
elif isinstance(file_or_filename, io.RawIOBase):
file = io.BufferedWriter(file_or_filename)
# Keep the original file open when the BufferedWriter is
# destroyed
stack.callback(file.detach)
else:
# This is to handle passed objects that aren't in the
# IOBase hierarchy, but just have a write method
file = io.BufferedIOBase()
file.writable = lambda: True
file.write = write
try:
# TextIOWrapper uses this methods to determine
# if BOM (for UTF-16, etc) should be added
file.seekable = file_or_filename.seekable
file.tell = file_or_filename.tell
except AttributeError:
pass
file = io.TextIOWrapper(file,
encoding=encoding,
errors='html_replace',
newline="\n")
# Keep the original file open when the TextIOWrapper is
# destroyed
stack.callback(file.detach)
yield file.write
ElementTree._get_writer = _get_writer
I've been monkeying around with this Zipstream module by SpiderOak that basically allows you to stream and zip file or folder without writing anything to disc. It yields chunks of irregularly sized data.
Now, I am trying to upload a directory to a file hosting site that requires me to send file and apikey fields inside a post request. With requests I have to build a dict for the apikey like so:
data = {'apikey': 'myapikey'}
and also read the entire zipstream into a string and pass it to a file-encoding dict:
files = {'file': ('mydir.zip', the_string_that_is_a_zipped_dir)}
then issue the request
r = requests.post(url, data=data, files=files).
This works ok. However I would like to transfer larger stuff in the future and reading the entire file in memory is a BAD idea to begin with.
I saw in the requests advanced section that you could send a generator as the data field, but then I can't send the api key and have to modify headers manually to set the content type and all that stuff so it doesn't work. Also tried to form a dictionary out of the api key and zip file generator like so
data = {
'file': ('mydir.zip', generator()),
'apikey': 'myapikey'
}
but this fails (as expected).
Is there a way to hack requests into using a generator that yields strings for a file in a multipart form-data?
Ok, after some struggle I managed to make this working without requests, instead using the module poster.
First I created a fileobject wrapper around zipstream like so:
from zipstream import ZipStream
class Zipit:
def __init__( self, path):
self.it = iter(ZipStream(path,compression=0))
self.next_chunk = ""
self.length = -1
self.path = path
self.__is_zipit__=''
#property
def size(self):
if self.length < 0:
self.length = 0
zip_object = ZipStream(self.path,compression=0)
for data in zip_object:
self.length += len(data)
return self.length
def growChunk( self ):
self.next_chunk = self.next_chunk + self.it.next()
def read( self, n ):
if self.next_chunk == None:
return None
try:
while len(self.next_chunk)<n:
self.growChunk()
rv = self.next_chunk[:n]
self.next_chunk = self.next_chunk[n:]
return rv
except StopIteration:
rv = self.next_chunk
self.next_chunk = None
return rv
in order to have an easy api (code shamelessly adapted from another example on SO).
Then as per poster's doc create the necessary multipart objects:
z = Zipit('/my/path/to/zip')
f = MultipartParam('file', fileobj=z, filesize=z.size, filename='test.zip',filetype='application/zip')
datagen, headers = multipart_encode([ f, ('akey', 'mykey')])
One last hack is skipping the reset in case the fileobject field is a Zipit instance:
def reset(self):
if hasattr(self.fileobj, '__is_zipit__'): return
if self.fileobj is not None:
self.fileobj.seek(0)
elif self.value is None:
raise ValueError("Don't know how to reset this parameter")
This worked for me. Hope it helps any of the five of you that read this.
I want to allow users to download an archive of multiple large files at once. However, the files and the archive may be too large to store in memory or on disk on my server (they are streamed in from other servers on the fly). I'd like to generate the archive as I stream it to the user.
I can use Tar or Zip or whatever is simplest. I am using django, which allows me to return a generator or file-like object in my response. This object could be used to pump the process along. However, I am having trouble figuring out how to build this sort of thing around the zipfile or tarfile libraries, and I'm afraid they may not support reading files as they go, or reading the archive as it is built.
This answer on converting an iterator to a file-like object might help. tarfile#addfile takes an iterable, but it appears to immediately pass that to shutil.copyfileobj, so this may not be as generator-friendly as I had hoped.
I ended up using SpiderOak ZipStream.
You can do it by generating and streaming a zip file with no compression, which is basically to just add the headers before each file's content. You're right, the libraries don't support this, but you can hack around them to get it working.
This code wraps zipfile.ZipFile with a class that manages the stream and creates instances of zipfile.ZipInfo for the files as they come. CRC and size can be set at the end. You can push data from the input stream into it with put_file(), write() and flush(), and read data out of it to the output stream with read().
import struct
import zipfile
import time
from StringIO import StringIO
class ZipStreamer(object):
def __init__(self):
self.out_stream = StringIO()
# write to the stringIO with no compression
self.zipfile = zipfile.ZipFile(self.out_stream, 'w', zipfile.ZIP_STORED)
self.current_file = None
self._last_streamed = 0
def put_file(self, name, date_time=None):
if date_time is None:
date_time = time.localtime(time.time())[:6]
zinfo = zipfile.ZipInfo(name, date_time)
zinfo.compress_type = zipfile.ZIP_STORED
zinfo.flag_bits = 0x08
zinfo.external_attr = 0600 << 16
zinfo.header_offset = self.out_stream.pos
# write right values later
zinfo.CRC = 0
zinfo.file_size = 0
zinfo.compress_size = 0
self.zipfile._writecheck(zinfo)
# write header to stream
self.out_stream.write(zinfo.FileHeader())
self.current_file = zinfo
def flush(self):
zinfo = self.current_file
self.out_stream.write(struct.pack("<LLL", zinfo.CRC, zinfo.compress_size, zinfo.file_size))
self.zipfile.filelist.append(zinfo)
self.zipfile.NameToInfo[zinfo.filename] = zinfo
self.current_file = None
def write(self, bytes):
self.out_stream.write(bytes)
self.out_stream.flush()
zinfo = self.current_file
# update these...
zinfo.CRC = zipfile.crc32(bytes, zinfo.CRC) & 0xffffffff
zinfo.file_size += len(bytes)
zinfo.compress_size += len(bytes)
def read(self):
i = self.out_stream.pos
self.out_stream.seek(self._last_streamed)
bytes = self.out_stream.read()
self.out_stream.seek(i)
self._last_streamed = i
return bytes
def close(self):
self.zipfile.close()
Keep in mind that this code was just a quick proof of concept and I did no further development or testing once I decided to let the http server itself deal with this problem. A few things you should look into if you decide to use it is to check if nested folders are archived correctly, and filename encoding (which is always a pain with zip files anyway).
You can stream a ZipFile to a Pylons or Django response fileobj by wrapping the fileobj in something file-like that implements tell(). This will buffer each individual file in the zip in memory, but stream the zip itself. We use it to stream download a zip file full of images, so we never buffer more than a single image in memory.
This example streams to sys.stdout. For Pylons use response.body_file, for Django you can use the HttpResponse itself as a file.
import zipfile
import sys
class StreamFile(object):
def __init__(self, fileobj):
self.fileobj = fileobj
self.pos = 0
def write(self, str):
self.fileobj.write(str)
self.pos += len(str)
def tell(self):
return self.pos
def flush(self):
self.fileobj.flush()
# Wrap a stream so ZipFile can use it
out = StreamFile(sys.stdout)
z = zipfile.ZipFile(out, 'w', zipfile.ZIP_DEFLATED)
for i in range(5):
z.writestr("hello{0}.txt".format(i), "this is hello{0} contents\n".format(i) * 3)
z.close()
Here is the solution from Pedro Werneck (from above) but with a fix to avoid collecting all data in memory (read method is fixed a little bit):
class ZipStreamer(object):
def __init__(self):
self.out_stream = StringIO.StringIO()
# write to the stringIO with no compression
self.zipfile = zipfile.ZipFile(self.out_stream, 'w', zipfile.ZIP_STORED)
self.current_file = None
self._last_streamed = 0
def put_file(self, name, date_time=None):
if date_time is None:
date_time = time.localtime(time.time())[:6]
zinfo = zipfile.ZipInfo(name, date_time)
zinfo.compress_type = zipfile.ZIP_STORED
zinfo.flag_bits = 0x08
zinfo.external_attr = 0600 << 16
zinfo.header_offset = self.out_stream.pos
# write right values later
zinfo.CRC = 0
zinfo.file_size = 0
zinfo.compress_size = 0
self.zipfile._writecheck(zinfo)
# write header to mega_streamer
self.out_stream.write(zinfo.FileHeader())
self.current_file = zinfo
def flush(self):
zinfo = self.current_file
self.out_stream.write(
struct.pack("<LLL", zinfo.CRC, zinfo.compress_size,
zinfo.file_size))
self.zipfile.filelist.append(zinfo)
self.zipfile.NameToInfo[zinfo.filename] = zinfo
self.current_file = None
def write(self, bytes):
self.out_stream.write(bytes)
self.out_stream.flush()
zinfo = self.current_file
# update these...
zinfo.CRC = zipfile.crc32(bytes, zinfo.CRC) & 0xffffffff
zinfo.file_size += len(bytes)
zinfo.compress_size += len(bytes)
def read(self):
self.out_stream.seek(self._last_streamed)
bytes = self.out_stream.read()
self._last_streamed = 0
# cleaning up memory in each iteration
self.out_stream.seek(0)
self.out_stream.truncate()
self.out_stream.flush()
return bytes
def close(self):
self.zipfile.close()
then you can use stream_generator function as a stream for a zip file
def stream_generator(files_paths):
s = ZipStreamer()
for f in files_paths:
s.put_file(f)
with open(f) as _f:
s.write(_f.read())
s.flush()
yield s.read()
s.close()
example for Falcon:
class StreamZipEndpoint(object):
def on_get(self, req, resp):
files_pathes = [
'/path/to/file/1',
'/path/to/file/2',
]
zip_filename = 'output_filename.zip'
resp.content_type = 'application/zip'
resp.set_headers([
('Content-Disposition', 'attachment; filename="%s"' % (
zip_filename,))
])
resp.stream = stream_generator(files_pathes)
An option is to use stream-zip (full disclosure: written by me)
Amending its example slightly:
from datetime import datetime
from stream_zip import stream_zip, ZIP_64
def non_zipped_files():
modified_at = datetime.now()
perms = 0o600
# Hard coded in this example, but in real cases could
# for example yield data from a remote source
def file_1_data():
for i in range(0, 1000):
yield b'Some bytes'
def file_2_data():
for i in range(0, 1000):
yield b'Some bytes'
yield 'my-file-1.txt', modified_at, perms, ZIP64, file_1_data()
yield 'my-file-2.txt', modified_at, perms, ZIP64, file_2_data()
zipped_chunks = stream_zip(non_zipped_files())
# Can print each chunk, or return them to a client,
# say using Django's StreamingHttpResponse
for zipped_chunk in zipped_chunks:
print(zipped_chunk)