I am trying to call an API to receive a presigned url using a gamma endpoint and upload an excel file to S3 bucket. Seeing the following error:
Traceback (most recent call last):
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\urllib3\connectionpool.py", line 703, in urlopen
httplib_response = self._make_request(
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\urllib3\connectionpool.py", line 398, in _make_request
conn.request(method, url, **httplib_request_kw)
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\urllib3\connection.py", line 239, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\http\client.py", line 1255, in request
self._send_request(method, url, body, headers, encode_chunked)
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\http\client.py", line 1301, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\http\client.py", line 1250, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\http\client.py", line 1049, in _send_output
self.send(chunk)
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\http\client.py", line 971, in send
self.sock.sendall(data)
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\ssl.py", line 1204, in sendall
v = self.send(byte_view[count:])
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\ssl.py", line 1173, in send
return self._sslobj.write(data)
ConnectionResetError: [WinError 10054] An existing connection was forcibly closed by the remote host
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\requests\adapters.py", line 440, in send
resp = conn.urlopen(
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\urllib3\connectionpool.py", line 785, in urlopen
retries = retries.increment(
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\urllib3\util\retry.py", line 550, in increment
raise six.reraise(type(error), error, _stacktrace)
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\urllib3\packages\six.py", line 769, in reraise
raise value.with_traceback(tb)
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\urllib3\connectionpool.py", line 703, in urlopen
httplib_response = self._make_request(
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\urllib3\connectionpool.py", line 398, in _make_request
conn.request(method, url, **httplib_request_kw)
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\urllib3\connection.py", line 239, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\http\client.py", line 1255, in request
self._send_request(method, url, body, headers, encode_chunked)
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\http\client.py", line 1301, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\http\client.py", line 1250, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\http\client.py", line 1049, in _send_output
self.send(chunk)
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\http\client.py", line 971, in send
self.sock.sendall(data)
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\ssl.py", line 1204, in sendall
v = self.send(byte_view[count:])
File "C:\Users\AppData\Local\Programs\PythonCodingPack\lib\ssl.py", line 1173, in send
return self._sslobj.write(data)
urllib3.exceptions.ProtocolError: ('Connection aborted.', ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "c:/Users/Desktop/Life Jacket/file/Generate_presignedURL.py", line 53, in
http_response = requests.post(response['url'], data=response['fields'], files=files,headers=headers)
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\requests\api.py", line 117, in post
return request('post', url, data=data, json=json, **kwargs)
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\requests\api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\requests\sessions.py", line 529, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\requests\sessions.py", line 645, in send
r = adapter.send(request, **kwargs)
File "C:\Users\AppData\Roaming\Python\Python38\site-packages\requests\adapters.py", line 501, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None))
PS C:\Users\Desktop\Life Jacket\file>
import constants
import boto3
from botocore.exceptions import ClientError
import requests
import pandas
OBJECT_NAME_TO_UPLOAD = r"C:\Users\Desktop\Life Jacket\file\PART_Data_4202022.xlsx"
s3_client = boto3.client(
's3',
#config= config(signature_version = "s3v4"), #do we need to add this???
aws_access_key_id=constants.access_key,
aws_secret_access_key=constants.secret_access_key
)
#Generate the presigned URL
try :
response = s3_client.generate_presigned_post(
Bucket='http://1234567890.execute-api.us-east-1.amazonaws.com/gamma/abcde/getS3PreSignedUrl',
Key = OBJECT_NAME_TO_UPLOAD,
Fields=None,
Conditions=None,
ExpiresIn = 3600
)
except ClientError as e:
if e.response['Error']['Code'] == 'EntityAlreadyExists':
print("User already exists")
else:
print("Unexpected error: %s" % e)
print(response)
#Upload file to S3 using presigned URL
try :
files = {'input_file': open(OBJECT_NAME_TO_UPLOAD, "rb")}
headers={
"Accept-Encoding": "*",
"Connection": "keep-alive"
}
http_response = requests.post(response['url'], data=response['fields'], files=files,headers=headers)
except ClientError as e:
if e.http_response['Error']['Code'] == 'EntityAlreadyExists':
print("User already exists")
else:
print("Unexpected error: %s" % e)
print(http_response)
print(f'File upload status code: {http_response.status_code}')
print(http_response)
Related
I want to upload some files to the flask server through requests module of python.It works well when the file is small,but errors occur when the file is large.
The flask serve code:
from flask import Flask
app = Flask(__name__)
#app.route('/test',methods=['get', 'post'])
def upload():
return "Success"
the request code:
url = "http://localhost:5000/test"
response = requests.post(url, files={'file': open('1.mhd', 'rb') })
print(response.text)
When the file size is small,the response is Success,but when the file is large(about 100M),the error is:
Traceback (most recent call last):
File "D:\software\anaconda\envs\torch13\lib\site-packages\urllib3\connectionpool.py", line 710, in urlopen
chunked=chunked,
File "D:\software\anaconda\envs\torch13\lib\site-packages\urllib3\connectionpool.py", line 398, in _make_request
conn.request(method, url, **httplib_request_kw)
File "D:\software\anaconda\envs\torch13\lib\site-packages\urllib3\connection.py", line 239, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "D:\software\anaconda\envs\torch13\lib\http\client.py", line 1281, in request
self._send_request(method, url, body, headers, encode_chunked)
File "D:\software\anaconda\envs\torch13\lib\http\client.py", line 1327, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "D:\software\anaconda\envs\torch13\lib\http\client.py", line 1276, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "D:\software\anaconda\envs\torch13\lib\http\client.py", line 1075, in _send_output
self.send(chunk)
File "D:\software\anaconda\envs\torch13\lib\http\client.py", line 997, in send
self.sock.sendall(data)
ConnectionAbortedError: [WinError 10053] 你的主机中的软件中止了一个已建立的连接。
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "D:\software\anaconda\envs\torch13\lib\site-packages\requests\adapters.py", line 499, in send
timeout=timeout,
File "D:\software\anaconda\envs\torch13\lib\site-packages\urllib3\connectionpool.py", line 788, in urlopen
method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
File "D:\software\anaconda\envs\torch13\lib\site-packages\urllib3\util\retry.py", line 550, in increment
raise six.reraise(type(error), error, _stacktrace)
File "D:\software\anaconda\envs\torch13\lib\site-packages\urllib3\packages\six.py", line 769, in reraise
raise value.with_traceback(tb)
File "D:\software\anaconda\envs\torch13\lib\site-packages\urllib3\connectionpool.py", line 710, in urlopen
chunked=chunked,
File "D:\software\anaconda\envs\torch13\lib\site-packages\urllib3\connectionpool.py", line 398, in _make_request
conn.request(method, url, **httplib_request_kw)
File "D:\software\anaconda\envs\torch13\lib\site-packages\urllib3\connection.py", line 239, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "D:\software\anaconda\envs\torch13\lib\http\client.py", line 1281, in request
self._send_request(method, url, body, headers, encode_chunked)
File "D:\software\anaconda\envs\torch13\lib\http\client.py", line 1327, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "D:\software\anaconda\envs\torch13\lib\http\client.py", line 1276, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "D:\software\anaconda\envs\torch13\lib\http\client.py", line 1075, in _send_output
self.send(chunk)
File "D:\software\anaconda\envs\torch13\lib\http\client.py", line 997, in send
self.sock.sendall(data)
urllib3.exceptions.ProtocolError: ('Connection aborted.', ConnectionAbortedError(10053, '你的主机中的软件中止了一个已建立的连接。', None, 10053, None))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "E:/py_code/flask/test.py", line 53, in <module>
resp = session.post(url, headers=headers, data=form)
File "D:\software\anaconda\envs\torch13\lib\site-packages\requests\sessions.py", line 635, in post
return self.request("POST", url, data=data, json=json, **kwargs)
File "D:\software\anaconda\envs\torch13\lib\site-packages\requests\sessions.py", line 587, in request
resp = self.send(prep, **send_kwargs)
File "D:\software\anaconda\envs\torch13\lib\site-packages\requests\sessions.py", line 701, in send
r = adapter.send(request, **kwargs)
File "D:\software\anaconda\envs\torch13\lib\site-packages\requests\adapters.py", line 547, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', ConnectionAbortedError(10053, '你的主机中的软件中止了一个已建立的连接。', None, 10053, None))
Process finished with exit code 1
I'm testing my Python script and when server turned off my script crashes. How can I change this, to make connection again. Now, my script is crashing when it fails to connect to the server. Here is my script:
import urllib.request
import json
def connectToServer():
with urllib.request.urlopen("http://localhost:5000/user/connect") as url:
data = json.loads(url.read().decode())
Here is error:
Traceback (most recent call last):
File "C:\Users\Rostik\AppData\Local\Programs\Python\Python39\lib\urllib\request.py", line 1346, in do_open
h.request(req.get_method(), req.selector, req.data, headers,
File "C:\Users\Rostik\AppData\Local\Programs\Python\Python39\lib\http\client.py", line 1255, in request
self._send_request(method, url, body, headers, encode_chunked)
File "C:\Users\Rostik\AppData\Local\Programs\Python\Python39\lib\http\client.py", line 1301, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "C:\Users\Rostik\AppData\Local\Programs\Python\Python39\lib\http\client.py", line 1250, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "C:\Users\Rostik\AppData\Local\Programs\Python\Python39\lib\http\client.py", line 1010, in _send_output
self.send(msg)
File "C:\Users\Rostik\AppData\Local\Programs\Python\Python39\lib\http\client.py", line 950, in send
self.connect()
File "C:\Users\Rostik\AppData\Local\Programs\Python\Python39\lib\http\client.py", line 921, in connect
self.sock = self._create_connection(
File "C:\Users\Rostik\AppData\Local\Programs\Python\Python39\lib\socket.py", line 843, in create_connection
raise err
File "C:\Users\Rostik\AppData\Local\Programs\Python\Python39\lib\socket.py", line 831, in create_connection
sock.connect(sa)
ConnectionRefusedError: [WinError 10061] Подключение не установлено, т.к. конечный компьютер отверг запрос на подключение
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "e:\Projects\Hacker_Pro Python\main.py", line 4, in <module>
server.connectToServer()
File "e:\Projects\Hacker_Pro Python\server.py", line 8, in connectToServer
with urllib.request.urlopen("http://localhost:5000/user/connect") as url:
File "C:\Users\Rostik\AppData\Local\Programs\Python\Python39\lib\urllib\request.py", line 214, in urlopen
return opener.open(url, data, timeout)
File "C:\Users\Rostik\AppData\Local\Programs\Python\Python39\lib\urllib\request.py", line 517, in open
response = self._open(req, data)
File "C:\Users\Rostik\AppData\Local\Programs\Python\Python39\lib\urllib\request.py", line 534, in _open
result = self._call_chain(self.handle_open, protocol, protocol +
File "C:\Users\Rostik\AppData\Local\Programs\Python\Python39\lib\urllib\request.py", line 494, in _call_chain
result = func(*args)
File "C:\Users\Rostik\AppData\Local\Programs\Python\Python39\lib\urllib\request.py", line 1375, in http_open
return self.do_open(http.client.HTTPConnection, req)
File "C:\Users\Rostik\AppData\Local\Programs\Python\Python39\lib\urllib\request.py", line 1349, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error [WinError 10061] Подключение не установлено, т.к. конечный компьютер отверг запрос на подключение>
The python try statement may help you.
def connectToServer():
try:
with urllib.request.urlopen("http://localhost:5000/user/connect") as url:
data = json.loads(url.read().decode())
except ConnectionRefusedError as error:
print('connection was refused:\n{}'.format(error))
except:
print('could not connect to server')
I am working on a system for analyzing data of any size and format streamed by the users to my private cloud based on Google Cloud Storage. Do you have any ideas how can I allow them to stream big data? At the moment I use Django API and I do this in this way:
def upload_blob(source_file_name, destination_blob_name):
blob = bucket.blob(destination_blob_name)
blob.upload_from_filename(source_file_name)
print('File {} uploaded to {}.'.format(
source_file_name,
destination_blob_name))
It works correctly with small files however when I send for example large movie I get the error shown below. I am aware that this is not the optimal solution but I have no idea how can I solve this. As you can notice at the moment they send me requests with the blob format but with very large files it does not work. Do you have any ideas how can I solve my problem and send users data of any size to Google Cloud Storage?
Internal Server Error: /cloud/
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 672, in urlopen
chunked=chunked,
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 387, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py", line 1252, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py", line 1298, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py", line 1247, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py", line 1065, in _send_output
self.send(chunk)
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py", line 987, in send
self.sock.sendall(data)
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/ssl.py", line 1034, in sendall
v = self.send(byte_view[count:])
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/ssl.py", line 1003, in send
return self._sslobj.write(data)
socket.timeout: The write operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 720, in urlopen
method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
File "/usr/local/lib/python3.7/site-packages/urllib3/util/retry.py", line 400, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/usr/local/lib/python3.7/site-packages/urllib3/packages/six.py", line 734, in reraise
raise value.with_traceback(tb)
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 672, in urlopen
chunked=chunked,
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 387, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py", line 1252, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py", line 1298, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py", line 1247, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py", line 1065, in _send_output
self.send(chunk)
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py", line 987, in send
self.sock.sendall(data)
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/ssl.py", line 1034, in sendall
v = self.send(byte_view[count:])
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/ssl.py", line 1003, in send
return self._sslobj.write(data)
urllib3.exceptions.ProtocolError: ('Connection aborted.', timeout('The write operation timed out'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/django/core/handlers/exception.py", line 34, in inner
response = get_response(request)
File "/usr/local/lib/python3.7/site-packages/django/core/handlers/base.py", line 115, in _get_response
response = self.process_exception_by_middleware(e, request)
File "/usr/local/lib/python3.7/site-packages/django/core/handlers/base.py", line 113, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "/usr/local/lib/python3.7/site-packages/django/views/decorators/csrf.py", line 54, in wrapped_view
return view_func(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/django/views/generic/base.py", line 71, in view
return self.dispatch(request, *args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/rest_framework/views.py", line 505, in dispatch
response = self.handle_exception(exc)
File "/usr/local/lib/python3.7/site-packages/rest_framework/views.py", line 465, in handle_exception
self.raise_uncaught_exception(exc)
File "/usr/local/lib/python3.7/site-packages/rest_framework/views.py", line 476, in raise_uncaught_exception
raise exc
File "/usr/local/lib/python3.7/site-packages/rest_framework/views.py", line 502, in dispatch
response = handler(request, *args, **kwargs)
File "/mypath/backend/views.py", line 635, in post
'user/' + str(user_name) + '/' + str(file))
File "/mypath/backend/views.py", line 214, in upload_blob
blob.upload_from_filename(source_file_name)
File "/usr/local/lib/python3.7/site-packages/google/cloud/storage/blob.py", line 1318, in upload_from_filename
predefined_acl=predefined_acl,
File "/usr/local/lib/python3.7/site-packages/google/cloud/storage/blob.py", line 1263, in upload_from_file
client, file_obj, content_type, size, num_retries, predefined_acl
File "/usr/local/lib/python3.7/site-packages/google/cloud/storage/blob.py", line 1173, in _do_upload
client, stream, content_type, size, num_retries, predefined_acl
File "/usr/local/lib/python3.7/site-packages/google/cloud/storage/blob.py", line 1120, in _do_resumable_upload
response = upload.transmit_next_chunk(transport)
File "/usr/local/lib/python3.7/site-packages/google/resumable_media/requests/upload.py", line 425, in transmit_next_chunk
retry_strategy=self._retry_strategy,
File "/usr/local/lib/python3.7/site-packages/google/resumable_media/requests/_helpers.py", line 136, in http_request
return _helpers.wait_and_retry(func, RequestsMixin._get_status_code, retry_strategy)
File "/usr/local/lib/python3.7/site-packages/google/resumable_media/_helpers.py", line 150, in wait_and_retry
response = func()
File "/usr/local/lib/python3.7/site-packages/google/auth/transport/requests.py", line 216, in request
method, url, data=data, headers=request_headers, **kwargs
File "/usr/local/lib/python3.7/site-packages/requests/sessions.py", line 533, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python3.7/site-packages/requests/sessions.py", line 646, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python3.7/site-packages/requests/adapters.py", line 498, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', timeout('The write operation timed out'))
[24/Mar/2020 19:17:26] "POST /cloud/ HTTP/1.1" 500 20879
Have a look at Resumable uploads.
This option provides a resumable data transfer feature that lets you resume upload operations after a communication failure has interrupted the flow of data.
Especially useful if you are transferring large files, because the likelihood of a network interruption or some other transmission failures is high. In case of a failure, you do not have to restart large file uploads from the beginning when using this option.
I am trying to launch my script to PA and while it works locally it does not work on PA and I cannot figure out why. I have the paid version of PA so I can access this site that is not whitelisted. Here is my script:
with requests.Session() as session:
# initates the login session
if os.path.exists(document_report + '.pdf'):
# initates the login session
post = session.post(POST_LOGIN_URL, data=payload, headers = dict(referrer = 'https://www.example.com/loginpage'))
#This URL is the page you actually want to pull down with requests.
REQUEST_URL = f'https://www.example.com/{stuff}/{document}'
#gets the request URL
r = session.get(REQUEST_URL)
# saves to directory
with open('./digests/1.pdf', 'wb') as j:
j.write(r.content)
j.close()
else:
post = session.post(POST_LOGIN_URL, data=payload, headers = dict(referrer = 'https://www.example.com/loginpage'))
#This URL is the page you actually want to pull down with requests.
REQUEST_URL = f'https://www.example.com/{stuff}/{document}'
#gets the request URL
r = session.get(REQUEST_URL)
# saves to directory
with open(document_report + '.pdf', 'wb') as j:
j.write(r.content)
j.close()
When I run the code, I get the following message:
Traceback (most recent call last):
File "/usr/lib/python3.7/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/usr/lib/python3.7/site-packages/urllib3/connectionpool.py", line 384, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "/usr/lib/python3.7/site-packages/urllib3/connectionpool.py", line 380, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.7/http/client.py", line 1344, in getresponse
response.begin()
File "/usr/lib/python3.7/http/client.py", line 306, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.7/http/client.py", line 275, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3.7/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/usr/lib/python3.7/site-packages/urllib3/connectionpool.py", line 638, in urlopen
_stacktrace=sys.exc_info()[2])
File "/usr/lib/python3.7/site-packages/urllib3/util/retry.py", line 368, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/usr/lib/python3.7/site-packages/urllib3/packages/six.py", line 685, in reraise
raise value.with_traceback(tb)
File "/usr/lib/python3.7/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/usr/lib/python3.7/site-packages/urllib3/connectionpool.py", line 384, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "/usr/lib/python3.7/site-packages/urllib3/connectionpool.py", line 380, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.7/http/client.py", line 1344, in getresponse
response.begin()
File "/usr/lib/python3.7/http/client.py", line 306, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.7/http/client.py", line 275, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/legalaide/emailProcessorV2.py", line 567, in <module>
post = session.post(POST_LOGIN_URL, data=payload, headers = dict(referrer = 'https://www.example.com/loginpage'))
File "/usr/lib/python3.7/site-packages/requests/sessions.py", line 581, in post
return self.request('POST', url, data=data, json=json, **kwargs)
File "/usr/lib/python3.7/site-packages/requests/sessions.py", line 533, in request
resp = self.send(prep, **send_kwargs)
File "/usr/lib/python3.7/site-packages/requests/sessions.py", line 646, in send
r = adapter.send(request, **kwargs)
File "/usr/lib/python3.7/site-packages/requests/adapters.py", line 498, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
I tried it again locally, and it still works. Any insight into a solution would be amazing!
I'm trying to upload a file using a PUT REST Api, but I get a ConnectionResetError. I've tried using urllib.request.Request() with urllib.request.urlopen(), as well as requests.put().
It works correctly when I'm using cURL:
$ curl -X PUT http://localhost:5000/root.bar/test/1.0/jre -H 'Content-Type: application/java-archive' -H 'Content-Name: bfg-1.12.16.jar' -H 'Authorization: Basic cm9vdDphbHBpbmU=' -d #C:/Users/niklas/Desktop/bfg-1.12.16.jar
The important portion of the code:
headers = {'Content-Type': args.mime, 'Content-Name': args.name}
if args.auth:
headers['Authorization'] = build_basicauth(username, password)
url = args.apiurl.rstrip('/') + '/{}/{}/{}/{}'.format(*parts)
if not urllib.parse.urlparse(url).scheme:
url = 'https://' + url
if args.test:
command = ['curl', '-X', 'PUT', url]
for key, value in headers.items():
command += ['-H', '{}: {}'.format(key, value)]
command += ['-d', '#' + args.file.name]
print('$', ' '.join(map(shlex.quote, command)))
return 0
response = requests.put(url, data=args.file, headers=headers)
print(response)
What am I missing that cURL is doing?
(PS: I've also tried sending bytes instead of a file-like object using requests.put() by passing data=args.file.read() instead)
The full traceback:
$ python -m fatartifacts.web.cli http://localhost:5000 root.bar:test:1.0:jre ~/Desktop/bfg-1.12.16.jar -m application/java-archive -u root:alpine
Traceback (most recent call last):
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\urllib3\connectionpool.py", line 601, in urlopen
chunked=chunked)
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\urllib3\connectionpool.py", line 357, in _make_request
conn.request(method, url, **httplib_request_kw)
File "c:\users\niklas\appdata\local\programs\python\python36\Lib\http\client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "c:\users\niklas\appdata\local\programs\python\python36\Lib\http\client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "c:\users\niklas\appdata\local\programs\python\python36\Lib\http\client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "c:\users\niklas\appdata\local\programs\python\python36\Lib\http\client.py", line 1065, in _send_output
self.send(chunk)
File "c:\users\niklas\appdata\local\programs\python\python36\Lib\http\client.py", line 986, in send
self.sock.sendall(data)
ConnectionResetError: [WinError 10054] An existing connection was forcibly closed by the remote host
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\requests\adapters.py", line 440, in send
timeout=timeout
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\urllib3\connectionpool.py", line 639, in urlopen
_stacktrace=sys.exc_info()[2])
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\urllib3\util\retry.py", line 357, in increment
raise six.reraise(type(error), error, _stacktrace)
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\urllib3\packages\six.py", line 685, in reraise
raise value.with_traceback(tb)
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\urllib3\connectionpool.py", line 601, in urlopen
chunked=chunked)
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\urllib3\connectionpool.py", line 357, in _make_request
conn.request(method, url, **httplib_request_kw)
File "c:\users\niklas\appdata\local\programs\python\python36\Lib\http\client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "c:\users\niklas\appdata\local\programs\python\python36\Lib\http\client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "c:\users\niklas\appdata\local\programs\python\python36\Lib\http\client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "c:\users\niklas\appdata\local\programs\python\python36\Lib\http\client.py", line 1065, in _send_output
self.send(chunk)
File "c:\users\niklas\appdata\local\programs\python\python36\Lib\http\client.py", line 986, in send
self.sock.sendall(data)
urllib3.exceptions.ProtocolError: ('Connection aborted.', ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "c:\users\niklas\appdata\local\programs\python\python36\Lib\runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "c:\users\niklas\appdata\local\programs\python\python36\Lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "C:\Users\niklas\repos\fatartifacts\fatartifacts\web\cli.py", line 106, in <module>
main_and_exit()
File "C:\Users\niklas\repos\fatartifacts\fatartifacts\web\cli.py", line 102, in main_and_exit
sys.exit(main())
File "C:\Users\niklas\repos\fatartifacts\fatartifacts\web\cli.py", line 97, in main
response = requests.put(url, data=args.file, headers=headers)
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\requests\api.py", line 126, in put
return request('put', url, data=data, **kwargs)
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\requests\api.py", line 58, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\requests\sessions.py", line 508, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\requests\sessions.py", line 618, in send
r = adapter.send(request, **kwargs)
File "C:\Users\niklas\.virtualenvs\fatartifacts-LoWBpE4v\lib\site-packages\requests\adapters.py", line 490, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None))
The REST API is implemented with flask and flask-restful. This exception seems to happen when
issuing the request using Python (urllib.request, requests)
the API returns a non-200 status code using flask.abort()
The issue was that the Flask REST API didn't consume all the content that was being sent with the request. The Flask application called abort(403) before the flask.request.stream was read.
Since I don't want to read all the data when the request is being rejected anyway, I found out that calling request.environ['wsgi.input'].close() closes the stream without reading it's content first, and this avoids the ConnectionResetError on the client-side.
def close_input_stream(func):
#functools.wraps(func)
def wrapper(*a, **kw):
try:
return func(*a, **kw)
finally:
fp = request.environ.get('wsgi.input')
if fp:
fp.close()
return wrapper