I'm trying to upload a file using a PUT REST Api, but I get a ConnectionResetError. I've tried using urllib.request.Request() with urllib.request.urlopen(), as well as requests.put().
It works correctly when I'm using cURL:
$ curl -X PUT http://localhost:5000/root.bar/test/1.0/jre -H 'Content-Type: application/java-archive' -H 'Content-Name: bfg-1.12.16.jar' -H 'Authorization: Basic cm9vdDphbHBpbmU=' -d #C:/Users/niklas/Desktop/bfg-1.12.16.jar
The important portion of the code:
headers = {'Content-Type': args.mime, 'Content-Name': args.name}
if args.auth:
headers['Authorization'] = build_basicauth(username, password)
url = args.apiurl.rstrip('/') + '/{}/{}/{}/{}'.format(*parts)
if not urllib.parse.urlparse(url).scheme:
url = 'https://' + url
if args.test:
command = ['curl', '-X', 'PUT', url]
for key, value in headers.items():
command += ['-H', '{}: {}'.format(key, value)]
command += ['-d', '#' + args.file.name]
print('$', ' '.join(map(shlex.quote, command)))
return 0
response = requests.put(url, data=args.file, headers=headers)
print(response)
What am I missing that cURL is doing?
(PS: I've also tried sending bytes instead of a file-like object using requests.put() by passing data=args.file.read() instead)
The full traceback:
$ python -m fatartifacts.web.cli http://localhost:5000 root.bar:test:1.0:jre ~/Desktop/bfg-1.12.16.jar -m application/java-archive -u root:alpine
Traceback (most recent call last):
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\urllib3\connectionpool.py", line 601, in urlopen
chunked=chunked)
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\urllib3\connectionpool.py", line 357, in _make_request
conn.request(method, url, **httplib_request_kw)
File "c:\users\niklas\appdata\local\programs\python\python36\Lib\http\client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "c:\users\niklas\appdata\local\programs\python\python36\Lib\http\client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "c:\users\niklas\appdata\local\programs\python\python36\Lib\http\client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "c:\users\niklas\appdata\local\programs\python\python36\Lib\http\client.py", line 1065, in _send_output
self.send(chunk)
File "c:\users\niklas\appdata\local\programs\python\python36\Lib\http\client.py", line 986, in send
self.sock.sendall(data)
ConnectionResetError: [WinError 10054] An existing connection was forcibly closed by the remote host
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\requests\adapters.py", line 440, in send
timeout=timeout
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\urllib3\connectionpool.py", line 639, in urlopen
_stacktrace=sys.exc_info()[2])
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\urllib3\util\retry.py", line 357, in increment
raise six.reraise(type(error), error, _stacktrace)
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\urllib3\packages\six.py", line 685, in reraise
raise value.with_traceback(tb)
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\urllib3\connectionpool.py", line 601, in urlopen
chunked=chunked)
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\urllib3\connectionpool.py", line 357, in _make_request
conn.request(method, url, **httplib_request_kw)
File "c:\users\niklas\appdata\local\programs\python\python36\Lib\http\client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "c:\users\niklas\appdata\local\programs\python\python36\Lib\http\client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "c:\users\niklas\appdata\local\programs\python\python36\Lib\http\client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "c:\users\niklas\appdata\local\programs\python\python36\Lib\http\client.py", line 1065, in _send_output
self.send(chunk)
File "c:\users\niklas\appdata\local\programs\python\python36\Lib\http\client.py", line 986, in send
self.sock.sendall(data)
urllib3.exceptions.ProtocolError: ('Connection aborted.', ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "c:\users\niklas\appdata\local\programs\python\python36\Lib\runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "c:\users\niklas\appdata\local\programs\python\python36\Lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "C:\Users\niklas\repos\fatartifacts\fatartifacts\web\cli.py", line 106, in <module>
main_and_exit()
File "C:\Users\niklas\repos\fatartifacts\fatartifacts\web\cli.py", line 102, in main_and_exit
sys.exit(main())
File "C:\Users\niklas\repos\fatartifacts\fatartifacts\web\cli.py", line 97, in main
response = requests.put(url, data=args.file, headers=headers)
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\requests\api.py", line 126, in put
return request('put', url, data=data, **kwargs)
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\requests\api.py", line 58, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\requests\sessions.py", line 508, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\requests\sessions.py", line 618, in send
r = adapter.send(request, **kwargs)
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\requests\adapters.py", line 490, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None))
The REST API is implemented with flask and flask-restful. This exception seems to happen when
issuing the request using Python (urllib.request, requests)
the API returns a non-200 status code using flask.abort()
The issue was that the Flask REST API didn't consume all the content that was being sent with the request. The Flask application called abort(403) before the flask.request.stream was read.
Since I don't want to read all the data when the request is being rejected anyway, I found out that calling request.environ['wsgi.input'].close() closes the stream without reading it's content first, and this avoids the ConnectionResetError on the client-side.
def close_input_stream(func):
#functools.wraps(func)
def wrapper(*a, **kw):
try:
return func(*a, **kw)
finally:
fp = request.environ.get('wsgi.input')
if fp:
fp.close()
return wrapper
Related
I want to upload some files to the flask server through requests module of python.It works well when the file is small,but errors occur when the file is large.
The flask serve code:
from flask import Flask
app = Flask(__name__)
#app.route('/test',methods=['get', 'post'])
def upload():
return "Success"
the request code:
url = "http://localhost:5000/test"
response = requests.post(url, files={'file': open('1.mhd', 'rb') })
print(response.text)
When the file size is small,the response is Success,but when the file is large(about 100M),the error is:
Traceback (most recent call last):
File "D:\software\anaconda\envs\torch13\lib\site-packages\urllib3\connectionpool.py", line 710, in urlopen
chunked=chunked,
File "D:\software\anaconda\envs\torch13\lib\site-packages\urllib3\connectionpool.py", line 398, in _make_request
conn.request(method, url, **httplib_request_kw)
File "D:\software\anaconda\envs\torch13\lib\site-packages\urllib3\connection.py", line 239, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "D:\software\anaconda\envs\torch13\lib\http\client.py", line 1281, in request
self._send_request(method, url, body, headers, encode_chunked)
File "D:\software\anaconda\envs\torch13\lib\http\client.py", line 1327, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "D:\software\anaconda\envs\torch13\lib\http\client.py", line 1276, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "D:\software\anaconda\envs\torch13\lib\http\client.py", line 1075, in _send_output
self.send(chunk)
File "D:\software\anaconda\envs\torch13\lib\http\client.py", line 997, in send
self.sock.sendall(data)
ConnectionAbortedError: [WinError 10053] 你的主机中的软件中止了一个已建立的连接。
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "D:\software\anaconda\envs\torch13\lib\site-packages\requests\adapters.py", line 499, in send
timeout=timeout,
File "D:\software\anaconda\envs\torch13\lib\site-packages\urllib3\connectionpool.py", line 788, in urlopen
method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
File "D:\software\anaconda\envs\torch13\lib\site-packages\urllib3\util\retry.py", line 550, in increment
raise six.reraise(type(error), error, _stacktrace)
File "D:\software\anaconda\envs\torch13\lib\site-packages\urllib3\packages\six.py", line 769, in reraise
raise value.with_traceback(tb)
File "D:\software\anaconda\envs\torch13\lib\site-packages\urllib3\connectionpool.py", line 710, in urlopen
chunked=chunked,
File "D:\software\anaconda\envs\torch13\lib\site-packages\urllib3\connectionpool.py", line 398, in _make_request
conn.request(method, url, **httplib_request_kw)
File "D:\software\anaconda\envs\torch13\lib\site-packages\urllib3\connection.py", line 239, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "D:\software\anaconda\envs\torch13\lib\http\client.py", line 1281, in request
self._send_request(method, url, body, headers, encode_chunked)
File "D:\software\anaconda\envs\torch13\lib\http\client.py", line 1327, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "D:\software\anaconda\envs\torch13\lib\http\client.py", line 1276, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "D:\software\anaconda\envs\torch13\lib\http\client.py", line 1075, in _send_output
self.send(chunk)
File "D:\software\anaconda\envs\torch13\lib\http\client.py", line 997, in send
self.sock.sendall(data)
urllib3.exceptions.ProtocolError: ('Connection aborted.', ConnectionAbortedError(10053, '你的主机中的软件中止了一个已建立的连接。', None, 10053, None))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "E:/py_code/flask/test.py", line 53, in <module>
resp = session.post(url, headers=headers, data=form)
File "D:\software\anaconda\envs\torch13\lib\site-packages\requests\sessions.py", line 635, in post
return self.request("POST", url, data=data, json=json, **kwargs)
File "D:\software\anaconda\envs\torch13\lib\site-packages\requests\sessions.py", line 587, in request
resp = self.send(prep, **send_kwargs)
File "D:\software\anaconda\envs\torch13\lib\site-packages\requests\sessions.py", line 701, in send
r = adapter.send(request, **kwargs)
File "D:\software\anaconda\envs\torch13\lib\site-packages\requests\adapters.py", line 547, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', ConnectionAbortedError(10053, '你的主机中的软件中止了一个已建立的连接。', None, 10053, None))
Process finished with exit code 1
<body class="white-vertion black-bg">
<!-- Start Loader -->
<p>
<py-script>
import ssl
from urllib.request import urlopen
from bs4 import BeautifulSoup
context = ssl._create_unverified_context()
result = urlopen("https://blog.naver.com/PostList.naver?blogId=woong3164&categoryNo=0&from=postList", context=context)
bsObj = BeautifulSoup(result.read(), "html.parser")
</py-script>
</p>
I used py-script in HTML to do web scraping. However, this error occurred.
'JsException(PythonError: Traceback (most recent call last):
File "/lib/python3.10/urllib/request.py", line 1348, in do_open
h.request(req.get_method(), req.selector, req.data, headers,
File "/lib/python3.10/http/client.py", line 1282, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/lib/python3.10/http/client.py", line 1328, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/lib/python3.10/http/client.py", line 1277, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/lib/python3.10/http/client.py", line 1037, in _send_output
self.send(msg)
File "/lib/python3.10/http/client.py", line 975, in send
self.connect()
File "/lib/python3.10/http/client.py", line 1447, in connect
super().connect()
File "/lib/python3.10/http/client.py", line 941, in connect
self.sock = self._create_connection(
File "/lib/python3.10/socket.py", line 845, in create_connection
raise err
File "/lib/python3.10/socket.py", line 833, in create_connection
sock.connect(sa)
BlockingIOError: [Errno 26] Operation in progress
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/lib/python3.10/site-packages/_pyodide/_base.py", line 429, in eval_code
.run(globals, locals)
File "/lib/python3.10/site-packages/_pyodide/_base.py", line 300, in run
coroutine = eval(self.code, globals, locals)
File "", line 6, in
File "/lib/python3.10/urllib/request.py", line 216, in urlopen
return opener.open(url, data, timeout)
File "/lib/python3.10/urllib/request.py", line 519, in open
response = self._open(req, data)
File "/lib/python3.10/urllib/request.py", line 536, in _open
result = self._call_chain(self.handle_open, protocol, protocol +
File "/lib/python3.10/urllib/request.py", line 496, in _call_chain
result = func(*args)
File "/lib/python3.10/urllib/request.py", line 1391, in https_open
return self.do_open(http.client.HTTPSConnection, req,
File "/lib/python3.10/urllib/request.py", line 1351, in do_open
raise URLError(err) urllib.error.URLError: )'
I think this error was caused by ssl.
How can I solve this error?
Your problem is caused by using unsupported Python packages. The package urllib uses APIs (TCP Sockets) that do not exist in the browser. This is not a limitation of PyScript, no browser application can use socket-based APIs.
The solution is to use supported APIs such as fetch or pyfetch.
I am trying to call an API to receive a presigned url using a gamma endpoint and upload an excel file to S3 bucket. Seeing the following error:
Traceback (most recent call last):
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\urllib3\connectionpool.py", line 703, in urlopen
httplib_response = self._make_request(
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\urllib3\connectionpool.py", line 398, in _make_request
conn.request(method, url, **httplib_request_kw)
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\urllib3\connection.py", line 239, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\http\client.py", line 1255, in request
self._send_request(method, url, body, headers, encode_chunked)
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\http\client.py", line 1301, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\http\client.py", line 1250, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\http\client.py", line 1049, in _send_output
self.send(chunk)
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\http\client.py", line 971, in send
self.sock.sendall(data)
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\ssl.py", line 1204, in sendall
v = self.send(byte_view[count:])
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\ssl.py", line 1173, in send
return self._sslobj.write(data)
ConnectionResetError: [WinError 10054] An existing connection was forcibly closed by the remote host
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\requests\adapters.py", line 440, in send
resp = conn.urlopen(
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\urllib3\connectionpool.py", line 785, in urlopen
retries = retries.increment(
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\urllib3\util\retry.py", line 550, in increment
raise six.reraise(type(error), error, _stacktrace)
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\urllib3\packages\six.py", line 769, in reraise
raise value.with_traceback(tb)
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\urllib3\connectionpool.py", line 703, in urlopen
httplib_response = self._make_request(
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\urllib3\connectionpool.py", line 398, in _make_request
conn.request(method, url, **httplib_request_kw)
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\urllib3\connection.py", line 239, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\http\client.py", line 1255, in request
self._send_request(method, url, body, headers, encode_chunked)
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\http\client.py", line 1301, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\http\client.py", line 1250, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\http\client.py", line 1049, in _send_output
self.send(chunk)
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\http\client.py", line 971, in send
self.sock.sendall(data)
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\ssl.py", line 1204, in sendall
v = self.send(byte_view[count:])
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\ssl.py", line 1173, in send
return self._sslobj.write(data)
urllib3.exceptions.ProtocolError: ('Connection aborted.', ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "c:/Users/Desktop/Life Jacket/file/Generate_presignedURL.py", line 53, in
http_response = requests.post(response['url'], data=response['fields'], files=files,headers=headers)
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\requests\api.py", line 117, in post
return request('post', url, data=data, json=json, **kwargs)
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\requests\api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\requests\sessions.py", line 529, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\requests\sessions.py", line 645, in send
r = adapter.send(request, **kwargs)
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\requests\adapters.py", line 501, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None))
PS C:\Users\Desktop\Life Jacket\file>
import constants
import boto3
from botocore.exceptions import ClientError
import requests
import pandas
OBJECT_NAME_TO_UPLOAD = r"C:\Users\Desktop\Life Jacket\file\PART_Data_4202022.xlsx"
s3_client = boto3.client(
's3',
#config= config(signature_version = "s3v4"), #do we need to add this???
aws_access_key_id=constants.access_key,
aws_secret_access_key=constants.secret_access_key
)
#Generate the presigned URL
try :
response = s3_client.generate_presigned_post(
Bucket='http://1234567890.execute-api.us-east-1.amazonaws.com/gamma/abcde/getS3PreSignedUrl',
Key = OBJECT_NAME_TO_UPLOAD,
Fields=None,
Conditions=None,
ExpiresIn = 3600
)
except ClientError as e:
if e.response['Error']['Code'] == 'EntityAlreadyExists':
print("User already exists")
else:
print("Unexpected error: %s" % e)
print(response)
#Upload file to S3 using presigned URL
try :
files = {'input_file': open(OBJECT_NAME_TO_UPLOAD, "rb")}
headers={
"Accept-Encoding": "*",
"Connection": "keep-alive"
}
http_response = requests.post(response['url'], data=response['fields'], files=files,headers=headers)
except ClientError as e:
if e.http_response['Error']['Code'] == 'EntityAlreadyExists':
print("User already exists")
else:
print("Unexpected error: %s" % e)
print(http_response)
print(f'File upload status code: {http_response.status_code}')
print(http_response)
I am working on a system for analyzing data of any size and format streamed by the users to my private cloud based on Google Cloud Storage. Do you have any ideas how can I allow them to stream big data? At the moment I use Django API and I do this in this way:
def upload_blob(source_file_name, destination_blob_name):
blob = bucket.blob(destination_blob_name)
blob.upload_from_filename(source_file_name)
print('File {} uploaded to {}.'.format(
source_file_name,
destination_blob_name))
It works correctly with small files however when I send for example large movie I get the error shown below. I am aware that this is not the optimal solution but I have no idea how can I solve this. As you can notice at the moment they send me requests with the blob format but with very large files it does not work. Do you have any ideas how can I solve my problem and send users data of any size to Google Cloud Storage?
Internal Server Error: /cloud/
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 672, in urlopen
chunked=chunked,
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 387, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py", line 1252, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py", line 1298, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py", line 1247, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py", line 1065, in _send_output
self.send(chunk)
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py", line 987, in send
self.sock.sendall(data)
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/ssl.py", line 1034, in sendall
v = self.send(byte_view[count:])
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/ssl.py", line 1003, in send
return self._sslobj.write(data)
socket.timeout: The write operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 720, in urlopen
method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
File "/usr/local/lib/python3.7/site-packages/urllib3/util/retry.py", line 400, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/usr/local/lib/python3.7/site-packages/urllib3/packages/six.py", line 734, in reraise
raise value.with_traceback(tb)
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 672, in urlopen
chunked=chunked,
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 387, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py", line 1252, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py", line 1298, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py", line 1247, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py", line 1065, in _send_output
self.send(chunk)
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py", line 987, in send
self.sock.sendall(data)
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/ssl.py", line 1034, in sendall
v = self.send(byte_view[count:])
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/ssl.py", line 1003, in send
return self._sslobj.write(data)
urllib3.exceptions.ProtocolError: ('Connection aborted.', timeout('The write operation timed out'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/django/core/handlers/exception.py", line 34, in inner
response = get_response(request)
File "/usr/local/lib/python3.7/site-packages/django/core/handlers/base.py", line 115, in _get_response
response = self.process_exception_by_middleware(e, request)
File "/usr/local/lib/python3.7/site-packages/django/core/handlers/base.py", line 113, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "/usr/local/lib/python3.7/site-packages/django/views/decorators/csrf.py", line 54, in wrapped_view
return view_func(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/django/views/generic/base.py", line 71, in view
return self.dispatch(request, *args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/rest_framework/views.py", line 505, in dispatch
response = self.handle_exception(exc)
File "/usr/local/lib/python3.7/site-packages/rest_framework/views.py", line 465, in handle_exception
self.raise_uncaught_exception(exc)
File "/usr/local/lib/python3.7/site-packages/rest_framework/views.py", line 476, in raise_uncaught_exception
raise exc
File "/usr/local/lib/python3.7/site-packages/rest_framework/views.py", line 502, in dispatch
response = handler(request, *args, **kwargs)
File "/mypath/backend/views.py", line 635, in post
'user/' + str(user_name) + '/' + str(file))
File "/mypath/backend/views.py", line 214, in upload_blob
blob.upload_from_filename(source_file_name)
File "/usr/local/lib/python3.7/site-packages/google/cloud/storage/blob.py", line 1318, in upload_from_filename
predefined_acl=predefined_acl,
File "/usr/local/lib/python3.7/site-packages/google/cloud/storage/blob.py", line 1263, in upload_from_file
client, file_obj, content_type, size, num_retries, predefined_acl
File "/usr/local/lib/python3.7/site-packages/google/cloud/storage/blob.py", line 1173, in _do_upload
client, stream, content_type, size, num_retries, predefined_acl
File "/usr/local/lib/python3.7/site-packages/google/cloud/storage/blob.py", line 1120, in _do_resumable_upload
response = upload.transmit_next_chunk(transport)
File "/usr/local/lib/python3.7/site-packages/google/resumable_media/requests/upload.py", line 425, in transmit_next_chunk
retry_strategy=self._retry_strategy,
File "/usr/local/lib/python3.7/site-packages/google/resumable_media/requests/_helpers.py", line 136, in http_request
return _helpers.wait_and_retry(func, RequestsMixin._get_status_code, retry_strategy)
File "/usr/local/lib/python3.7/site-packages/google/resumable_media/_helpers.py", line 150, in wait_and_retry
response = func()
File "/usr/local/lib/python3.7/site-packages/google/auth/transport/requests.py", line 216, in request
method, url, data=data, headers=request_headers, **kwargs
File "/usr/local/lib/python3.7/site-packages/requests/sessions.py", line 533, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python3.7/site-packages/requests/sessions.py", line 646, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python3.7/site-packages/requests/adapters.py", line 498, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', timeout('The write operation timed out'))
[24/Mar/2020 19:17:26] "POST /cloud/ HTTP/1.1" 500 20879
Have a look at Resumable uploads.
This option provides a resumable data transfer feature that lets you resume upload operations after a communication failure has interrupted the flow of data.
Especially useful if you are transferring large files, because the likelihood of a network interruption or some other transmission failures is high. In case of a failure, you do not have to restart large file uploads from the beginning when using this option.
I am trying to launch my script to PA and while it works locally it does not work on PA and I cannot figure out why. I have the paid version of PA so I can access this site that is not whitelisted. Here is my script:
with requests.Session() as session:
# initates the login session
if os.path.exists(document_report + '.pdf'):
# initates the login session
post = session.post(POST_LOGIN_URL, data=payload, headers = dict(referrer = 'https://www.example.com/loginpage'))
#This URL is the page you actually want to pull down with requests.
REQUEST_URL = f'https://www.example.com/{stuff}/{document}'
#gets the request URL
r = session.get(REQUEST_URL)
# saves to directory
with open('./digests/1.pdf', 'wb') as j:
j.write(r.content)
j.close()
else:
post = session.post(POST_LOGIN_URL, data=payload, headers = dict(referrer = 'https://www.example.com/loginpage'))
#This URL is the page you actually want to pull down with requests.
REQUEST_URL = f'https://www.example.com/{stuff}/{document}'
#gets the request URL
r = session.get(REQUEST_URL)
# saves to directory
with open(document_report + '.pdf', 'wb') as j:
j.write(r.content)
j.close()
When I run the code, I get the following message:
Traceback (most recent call last):
File "/usr/lib/python3.7/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/usr/lib/python3.7/site-packages/urllib3/connectionpool.py", line 384, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "/usr/lib/python3.7/site-packages/urllib3/connectionpool.py", line 380, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.7/http/client.py", line 1344, in getresponse
response.begin()
File "/usr/lib/python3.7/http/client.py", line 306, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.7/http/client.py", line 275, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3.7/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/usr/lib/python3.7/site-packages/urllib3/connectionpool.py", line 638, in urlopen
_stacktrace=sys.exc_info()[2])
File "/usr/lib/python3.7/site-packages/urllib3/util/retry.py", line 368, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/usr/lib/python3.7/site-packages/urllib3/packages/six.py", line 685, in reraise
raise value.with_traceback(tb)
File "/usr/lib/python3.7/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/usr/lib/python3.7/site-packages/urllib3/connectionpool.py", line 384, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "/usr/lib/python3.7/site-packages/urllib3/connectionpool.py", line 380, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.7/http/client.py", line 1344, in getresponse
response.begin()
File "/usr/lib/python3.7/http/client.py", line 306, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.7/http/client.py", line 275, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/legalaide/emailProcessorV2.py", line 567, in <module>
post = session.post(POST_LOGIN_URL, data=payload, headers = dict(referrer = 'https://www.example.com/loginpage'))
File "/usr/lib/python3.7/site-packages/requests/sessions.py", line 581, in post
return self.request('POST', url, data=data, json=json, **kwargs)
File "/usr/lib/python3.7/site-packages/requests/sessions.py", line 533, in request
resp = self.send(prep, **send_kwargs)
File "/usr/lib/python3.7/site-packages/requests/sessions.py", line 646, in send
r = adapter.send(request, **kwargs)
File "/usr/lib/python3.7/site-packages/requests/adapters.py", line 498, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
I tried it again locally, and it still works. Any insight into a solution would be amazing!