My AWS lambda connection keeps timing out due to socket errors. I have set out the timeout for the lambda to be 12 minutes in the code (EDIT: I mean in the lambda console) and the lambda is not in private vpc so the problem is not related to that.
For example: I run a function that runs 450 seconds. The function runs successfully in the AWS but it is above some magical threshold so the python code waits until the timeout and then throws an error. If the lambda runtime is less than the magical threshold then the python code works without any errors.
Might it be some kind of socket error that then loses the connection after some time but keeps waiting until the socket times out? I have no experience related to how sockets work in Ubuntu machines.
My code for testing:
Lambda code:
import json
from time import sleep
def lambda_handler(event, context):
print("SLEEPING: " + str(event["sleep_time"]) + "s")
sleep(event["sleep_time"])
print("SLEEP ENDED")
return {
"statusCode": 200,
"sleep_time": event["sleep_time"]
}
Code to invoke lambda:
import boto3
from botocore.client import Config
import json
from datetime import datetime
read_timeout = 500
connect_timeout = 500
sleep_time = 450
config = Config(
read_timeout=read_timeout,
connect_timeout=connect_timeout,
retries={"max_attempts": 0}
)
lambda_client = boto3.client(
"lambda",
aws_access_key_id="xxx",
aws_secret_access_key="xxx",
config=config
)
st = datetime.now()
print(f"STARTED AT: {st}")
print(f"\tSLEEP TIME PARAMETER: {sleep_time}")
print(f"\tCONFIG read_timeout {read_timeout}")
print(f"\tCONFIG connect_timeout {connect_timeout}")
try:
lambda_client.invoke(
FunctionName="helloWorld",
Payload=json.dumps({"sleep_time": sleep_time})
)
finally:
et = datetime.now()
print(f"ENDED AT: {et}")
print(f"SECONDS TAKEN TO COMPLETE: {et - st}")
The function works when the sleep_time is set to low e.q. 3 but I keep running into errors when I set the sleep_time to larger values.
Output with sleep_time=3
STARTED AT: 2022-08-02 18:17:04.015493
SLEEP TIME PARAMETER: 3
CONFIG read_timeout 500
CONFIG connect_timeout 500
ENDED AT: 2022-08-02 18:17:07.357435
SECONDS TAKEN TO COMPLETE: 0:00:03.341942
Output with longer sleeptime:
STARTED AT: 2022-08-02 19:03:36.196755
SLEEP TIME PARAMETER: 450
CONFIG read_timeout 500
CONFIG connect_timeout 500
ENDED AT: 2022-08-02 19:11:56.553257
SECONDS TAKEN TO COMPLETE: 0:08:20.356502
And I get the following error message:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 421, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 416, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.8/http/client.py", line 1348, in getresponse
response.begin()
File "/usr/lib/python3.8/http/client.py", line 316, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.8/http/client.py", line 277, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/lib/python3.8/socket.py", line 669, in readinto
return self._sock.recv_into(b)
File "/usr/lib/python3.8/ssl.py", line 1241, in recv_into
return self.read(nbytes, buffer)
File "/usr/lib/python3.8/ssl.py", line 1099, in read
return self._sslobj.read(len, buffer)
socket.timeout: The read operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/user/.local/lib/python3.8/site-packages/botocore/httpsession.py", line 448, in send
urllib_response = conn.urlopen(
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 719, in urlopen
retries = retries.increment(
File "/usr/lib/python3/dist-packages/urllib3/util/retry.py", line 376, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/usr/lib/python3/dist-packages/six.py", line 703, in reraise
raise value
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 665, in urlopen
httplib_response = self._make_request(
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 423, in _make_request
self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 330, in _raise_timeout
raise ReadTimeoutError(
urllib3.exceptions.ReadTimeoutError: AWSHTTPSConnectionPool(host='lambda.eu-west-1.amazonaws.com', port=443): Read timed out. (read timeout=900)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/tmp/lambda_test.py", line 24, in <module>
lambda_client.invoke(
File "/home/user/.local/lib/python3.8/site-packages/botocore/client.py", line 508, in _api_call
return self._make_api_call(operation_name, kwargs)
File "/home/user/.local/lib/python3.8/site-packages/botocore/client.py", line 898, in _make_api_call
http, parsed_response = self._make_request(
File "/home/user/.local/lib/python3.8/site-packages/botocore/client.py", line 921, in _make_request
return self._endpoint.make_request(operation_model, request_dict)
File "/home/user/.local/lib/python3.8/site-packages/botocore/endpoint.py", line 119, in make_request
return self._send_request(request_dict, operation_model)
File "/home/user/.local/lib/python3.8/site-packages/botocore/endpoint.py", line 202, in _send_request
while self._needs_retry(
File "/home/user/.local/lib/python3.8/site-packages/botocore/endpoint.py", line 354, in _needs_retry
responses = self._event_emitter.emit(
File "/home/user/.local/lib/python3.8/site-packages/botocore/hooks.py", line 412, in emit
return self._emitter.emit(aliased_event_name, **kwargs)
File "/home/user/.local/lib/python3.8/site-packages/botocore/hooks.py", line 256, in emit
return self._emit(event_name, kwargs)
File "/home/user/.local/lib/python3.8/site-packages/botocore/hooks.py", line 239, in _emit
response = handler(**kwargs)
File "/home/user/.local/lib/python3.8/site-packages/botocore/retryhandler.py", line 207, in __call__
if self._checker(**checker_kwargs):
File "/home/user/.local/lib/python3.8/site-packages/botocore/retryhandler.py", line 284, in __call__
should_retry = self._should_retry(
File "/home/user/.local/lib/python3.8/site-packages/botocore/retryhandler.py", line 320, in _should_retry
return self._checker(attempt_number, response, caught_exception)
File "/home/user/.local/lib/python3.8/site-packages/botocore/retryhandler.py", line 363, in __call__
checker_response = checker(
File "/home/user/.local/lib/python3.8/site-packages/botocore/retryhandler.py", line 247, in __call__
return self._check_caught_exception(
File "/home/user/.local/lib/python3.8/site-packages/botocore/retryhandler.py", line 416, in _check_caught_exception
raise caught_exception
File "/home/user/.local/lib/python3.8/site-packages/botocore/endpoint.py", line 281, in _do_get_response
http_response = self._send(request)
File "/home/user/.local/lib/python3.8/site-packages/botocore/endpoint.py", line 377, in _send
return self.http_session.send(request)
File "/home/user/.local/lib/python3.8/site-packages/botocore/httpsession.py", line 485, in send
raise ReadTimeoutError(endpoint_url=request.url, error=e)
botocore.exceptions.ReadTimeoutError: Read timeout on endpoint URL: "https://lambda.eu-west-1.amazonaws.com/2015-03-31/functions/helloWorld/invocations"
Edit:
Cloudwatch log for the errorcase:
2022-08-02T19:03:36.526+03:00 START RequestId: feffa401-62a2-4c77-8a92-96be27bdceeb Version: $LATEST
2022-08-02T19:03:36.529+03:00 SLEEPING: 450s
2022-08-02T19:11:06.638+03:00 SLEEP ENDED
2022-08-02T19:11:06.640+03:00 END RequestId: feffa401-62a2-4c77-8a92-96be27bdceeb
2022-08-02T19:11:06.640+03:00 REPORT RequestId: feffa401-62a2-4c77-8a92-96be27bdceeb Duration: 450102.24 ms Billed Duration: 450103 ms Memory Size: 128 MB Max Memory Used: 37 MB
The one that succeeded is similar to this
Related
I have a python script in a docker that runs and collects data from an api on a loop.
`for idx, items in enumerate(SearchList):
#get access token
access_token = GetPricingAccTok()
#request id to get results
reqid = RequestIdCompSearch(items, access_token)
#api to get results by doing a while loop appending results together till api returns 100% as completed percentage
Result = GetReqResults(reqid, access_token)
Pricings = GetPricingDict(Result)
filenamepricing = "pricinghistory/CorrectFormat/Pricings_Data_{idn}_{ts}.json".format(
idn = idx,
ts= dt.datetime.now().isoformat()
)
GetAzContainer().upload_blob(
name=filenamepricing,
data=json.dumps(obj= Pricings, indent=4),
blob_type='BlockBlob'
)
`
When i run and test this code locally on my windows machine it works well without an issue and completes the requests
but in the docker on azure container instance it then returns an error showing that the connection has timed out. Not sure what the difference is between the running it locally and in the docker that i then recieve this error.
I have tried addinng this code at the start
import socket
from urllib3.connection import HTTPConnection
HTTPConnection.default_socket_options =(HTTPConnection.default_socket_options + [
(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) #Enables the feature
,(socket.SOL_TCP, socket.TCP_KEEPIDLE, 45) #Overrides the time when the stack willl start sending KeppAlives after no data received on a Persistent Connection
,(socket.SOL_TCP, socket.TCP_KEEPINTVL, 10) #Defines how often thoe KA will be sent between them
,(socket.SOL_TCP, socket.TCP_KEEPCNT, 60) #How many attemps will your code try if the server goes down before droping the connection.
]
)
but it still times out. showing the below error
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 445, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 440, in _make_request
httplib_response = conn.getresponse()
File "/usr/local/lib/python3.8/http/client.py", line 1348, in getresponse
response.begin()
File "/usr/local/lib/python3.8/http/client.py", line 316, in begin
version, status, reason = self._read_status()
File "/usr/local/lib/python3.8/http/client.py", line 277, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/local/lib/python3.8/socket.py", line 669, in readinto
return self._sock.recv_into(b)
File "/usr/local/lib/python3.8/ssl.py", line 1241, in recv_into
return self.read(nbytes, buffer)
File "/usr/local/lib/python3.8/ssl.py", line 1099, in read
return self._sslobj.read(len, buffer)
TimeoutError: [Errno 110] Connection timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/requests/adapters.py", line 440, in send
resp = conn.urlopen(
File "/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 755, in urlopen
retries = retries.increment(
File "/usr/local/lib/python3.8/site-packages/urllib3/util/retry.py", line 532, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/usr/local/lib/python3.8/site-packages/urllib3/packages/six.py", line 770, in reraise
raise value
File "/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 447, in _make_request
self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
File "/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 353, in _raise_timeout
raise ReadTimeoutError(
urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='service.xxx.com', port=443): Read timed out. (read timeout=None)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "src/script.py", line 298, in <module>
reqid = RequestIdCompSearch(items, access_token)
File "src/script.py", line 111, in RequestIdCompSearch
InitResp =rq.post( ReqUrl, headers= ReqHead, data=Jsondata)
File "/usr/local/lib/python3.8/site-packages/requests/api.py", line 117, in post
return request('post', url, data=data, json=json, **kwargs)
File "/usr/local/lib/python3.8/site-packages/requests/api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/local/lib/python3.8/site-packages/requests/sessions.py", line 529, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python3.8/site-packages/requests/sessions.py", line 645, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python3.8/site-packages/requests/adapters.py", line 532, in send
raise ReadTimeout(e, request=request)
requests.exceptions.ReadTimeout: HTTPSConnectionPool(host='service.xxxx.com', port=443): Read timed out. (read timeout=None)
I have two telegram bots made with PyTelegramBotApi (not the best library, I know) deployed on Heroku. One big difference that matters: one is running with Threading to periodically send notifications to "subscribers" (around 20 people). And it crashes approximately every month with following exception trace in Heroku console:
2022-01-14T08:00:11.068553+00:00 app[worker.1]: File "/app/.heroku/python/lib/python3.9/site-packages/telebot/apihelper.py", line 139, in _make_request
2022-01-14T08:00:11.068739+00:00 app[worker.1]: result = _get_req_session().request(
2022-01-14T08:00:11.068748+00:00 app[worker.1]: File "/app/.heroku/python/lib/python3.9/site-packages/requests/sessions.py", line 542, in request
2022-01-14T08:00:11.068983+00:00 app[worker.1]: resp = self.send(prep, **send_kwargs)
2022-01-14T08:00:11.068994+00:00 app[worker.1]: File "/app/.heroku/python/lib/python3.9/site-packages/requests/sessions.py", line 655, in send
2022-01-14T08:00:11.069233+00:00 app[worker.1]: r = adapter.send(request, **kwargs)
2022-01-14T08:00:11.069242+00:00 app[worker.1]: File "/app/.heroku/python/lib/python3.9/site-packages/requests/adapters.py", line 529, in send
2022-01-14T08:00:11.069448+00:00 app[worker.1]: raise ReadTimeout(e, request=request)
2022-01-14T08:00:11.069478+00:00 app[worker.1]: requests.exceptions.ReadTimeout: HTTPSConnectionPool(host='api.telegram.org', port=443): Read timed out. (read timeout=25)
I didn't record previous exception traces (my bad), but I can remember it looking pretty much the same (with the most recent exception being ReadTimeout). Moreover, any of my scripts were never seen in this trace (and this makes me feel it's incomplete due to some Heroku log console bug). I myself don't use requests module directly. Also it definitely can't be caused by dyno sleep as I use only worker dyno (it never sleeps). Both this bot and the one not crashing are using Heroku Postgres.
Any ideas on what could be causing this exception?
EDIT:
full exception trace I accidentally got in debug mode while testing somethin else on my machine:
Traceback (most recent call last):
File "D:\Deadliner0307\lib\site-packages\urllib3\connectionpool.py", line 445, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "D:\Deadliner0307\lib\site-packages\urllib3\connectionpool.py", line 440, in _make_request
httplib_response = conn.getresponse()
File "C:\Users\vva07\AppData\Local\Programs\Python\Python39-32\lib\http\client.py", line 1347, in getresponse
response.begin()
File "C:\Users\vva07\AppData\Local\Programs\Python\Python39-32\lib\http\client.py", line 307, in begin
version, status, reason = self._read_status()
File "C:\Users\vva07\AppData\Local\Programs\Python\Python39-32\lib\http\client.py", line 268, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "C:\Users\vva07\AppData\Local\Programs\Python\Python39-32\lib\socket.py", line 704, in readinto
return self._sock.recv_into(b)
File "C:\Users\vva07\AppData\Local\Programs\Python\Python39-32\lib\ssl.py", line 1241, in recv_into
return self.read(nbytes, buffer)
File "C:\Users\vva07\AppData\Local\Programs\Python\Python39-32\lib\ssl.py", line 1099, in read
return self._sslobj.read(len, buffer)
socket.timeout: The read operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "D:\Deadliner0307\lib\site-packages\requests\adapters.py", line 439, in send
resp = conn.urlopen(
File "D:\Deadliner0307\lib\site-packages\urllib3\connectionpool.py", line 755, in urlopen
retries = retries.increment(
File "D:\Deadliner0307\lib\site-packages\urllib3\util\retry.py", line 532, in increment
raise six.reraise(type(error), error, _stacktrace)
File "D:\Deadliner0307\lib\site-packages\urllib3\packages\six.py", line 770, in reraise
raise value
File "D:\Deadliner0307\lib\site-packages\urllib3\connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "D:\Deadliner0307\lib\site-packages\urllib3\connectionpool.py", line 447, in _make_request
self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
File "D:\Deadliner0307\lib\site-packages\urllib3\connectionpool.py", line 336, in _raise_timeout
raise ReadTimeoutError(
urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='api.telegram.org', port=443): Read timed out. (read timeout=25)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\vva07\OneDrive\Документы\проекты\Deadliner0307\main.py", line 450, in <module>
bot.polling(none_stop=True, interval=1)
File "C:\Users\vva07\OneDrive\Документы\проекты\Deadliner0307\main.py", line 446, in deadliner0307
else:
File "D:\Deadliner0307\lib\site-packages\telebot\__init__.py", line 633, in polling
self.__threaded_polling(non_stop, interval, timeout, long_polling_timeout, allowed_updates)
File "D:\Deadliner0307\lib\site-packages\telebot\__init__.py", line 692, in __threaded_polling
raise e
File "D:\Deadliner0307\lib\site-packages\telebot\__init__.py", line 654, in __threaded_polling
polling_thread.raise_exceptions()
File "D:\Deadliner0307\lib\site-packages\telebot\util.py", line 100, in raise_exceptions
raise self.exception_info
File "D:\Deadliner0307\lib\site-packages\telebot\util.py", line 82, in run
task(*args, **kwargs)
File "D:\Deadliner0307\lib\site-packages\telebot\__init__.py", line 391, in __retrieve_updates
updates = self.get_updates(offset=(self.last_update_id + 1),
File "D:\Deadliner0307\lib\site-packages\telebot\__init__.py", line 371, in get_updates
json_updates = apihelper.get_updates(self.token, offset, limit, timeout, allowed_updates, long_polling_timeout)
File "D:\Deadliner0307\lib\site-packages\telebot\apihelper.py", line 312, in get_updates
return _make_request(token, method_url, params=payload)
File "D:\Deadliner0307\lib\site-packages\telebot\apihelper.py", line 139, in _make_request
result = _get_req_session().request(
File "D:\Deadliner0307\lib\site-packages\requests\sessions.py", line 542, in request
resp = self.send(prep, **send_kwargs)
File "D:\Deadliner0307\lib\site-packages\requests\sessions.py", line 655, in send
r = adapter.send(request, **kwargs)
File "D:\Deadliner0307\lib\site-packages\requests\adapters.py", line 529, in send
raise ReadTimeout(e, request=request)
requests.exceptions.ReadTimeout: HTTPSConnectionPool(host='api.telegram.org', port=443): Read timed out. (read timeout=25)
I have found a way to at least keep the bot working. Be careful: this may not fit your application (in your case restarting straight up after a specific exception may corrupt data or break something else). In main file, I made this construction, which basically restarts a bot after an unhandled exception, which would otherwise just stop the bot:
def exception_handler(count: int = 0):
"""Relaunching bot unless exceptions occur more than 2 times a day
(script is reset daily on Heroku)"""
if count < 3:
if count > 0:
print("An exception occurred, relaunching . . .")
time.sleep(5)
try:
deadliner0307() # bot main function with (bot.polling starts there)
except Exception as ex:
count += 1
# Notifying myself about exception via free Airbrake addon:
notifier = pybrake.Notifier(project_id=399289,
project_key='129d3450356965175fda762b69e1babf',
environment='production')
notifier.notify(ex)
exception_handler(count)
else:
print("Too much exceptions occurred, shutting down . . .")
if __name__ == '__main__':
exception_handler()
I have a kubernetes cluster (on GKE) which runs tornado.web.Application server, that calls heavy dask.distributed calculations on that cluster.
Now I want to test my server, and send it multiple requests at once, to see if it handles those requests as it should.
When I'm sending only one request, it does everything and behaves as expected, no matter how big the request is.
But when I send more than one request, after like 5 minutes (I also tried on server side just to sleep, without making any calculations and got the same results), I get the following:
Traceback (most recent call last):
File "/home/michael/anaconda3/lib/python3.7/site-packages/urllib3/connectionpool.py", line 706, in urlopen
chunked=chunked,
File "/home/michael/anaconda3/lib/python3.7/site-packages/urllib3/connectionpool.py", line 445, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/home/michael/anaconda3/lib/python3.7/site-packages/urllib3/connectionpool.py", line 440, in _make_request
httplib_response = conn.getresponse()
File "/home/michael/anaconda3/lib/python3.7/http/client.py", line 1354, in getresponse
response.begin()
File "/home/michael/anaconda3/lib/python3.7/http/client.py", line 306, in begin
version, status, reason = self._read_status()
File "/home/michael/anaconda3/lib/python3.7/http/client.py", line 267, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/home/michael/anaconda3/lib/python3.7/socket.py", line 589, in readinto
return self._sock.recv_into(b)
ConnectionResetError: [Errno 104] Connection reset by peer
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/michael/anaconda3/lib/python3.7/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/home/michael/anaconda3/lib/python3.7/site-packages/urllib3/connectionpool.py", line 756, in urlopen
method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
File "/home/michael/anaconda3/lib/python3.7/site-packages/urllib3/util/retry.py", line 531, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/home/michael/anaconda3/lib/python3.7/site-packages/urllib3/packages/six.py", line 734, in reraise
raise value.with_traceback(tb)
File "/home/michael/anaconda3/lib/python3.7/site-packages/urllib3/connectionpool.py", line 706, in urlopen
chunked=chunked,
File "/home/michael/anaconda3/lib/python3.7/site-packages/urllib3/connectionpool.py", line 445, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/home/michael/anaconda3/lib/python3.7/site-packages/urllib3/connectionpool.py", line 440, in _make_request
httplib_response = conn.getresponse()
File "/home/michael/anaconda3/lib/python3.7/http/client.py", line 1354, in getresponse
response.begin()
File "/home/michael/anaconda3/lib/python3.7/http/client.py", line 306, in begin
version, status, reason = self._read_status()
File "/home/michael/anaconda3/lib/python3.7/http/client.py", line 267, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/home/michael/anaconda3/lib/python3.7/socket.py", line 589, in readinto
return self._sock.recv_into(b)
urllib3.exceptions.ProtocolError: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/michael/anaconda3/lib/python3.7/threading.py", line 926, in _bootstrap_inner
self.run()
File "/home/michael/anaconda3/lib/python3.7/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "<ipython-input-4-ce4ad22684d3>", line 24, in fire
response = requests.post(url, data=payload, headers=headers, timeout=3600)
File "/home/michael/anaconda3/lib/python3.7/site-packages/requests/api.py", line 119, in post
return request('post', url, data=data, json=json, **kwargs)
File "/home/michael/anaconda3/lib/python3.7/site-packages/requests/api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "/home/michael/anaconda3/lib/python3.7/site-packages/requests/sessions.py", line 542, in request
resp = self.send(prep, **send_kwargs)
File "/home/michael/anaconda3/lib/python3.7/site-packages/requests/sessions.py", line 655, in send
r = adapter.send(request, **kwargs)
File "/home/michael/anaconda3/lib/python3.7/site-packages/requests/adapters.py", line 498, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer'))
And none of the requests returns.
How can I get the behaviour I need, without these errors?
I need a server that can handle many requests, and to test it from my computer.
I'm fetching OHLCV data from Binance via Binance api on AWS-EC2.
But sometime(once a couple of days) error "104, 'ECONNRESET'" occur and program stop.
The program written in Python3.7.3 and run on AWS-EC2
The code is following. But this code couldn't continue.
I want to continue the program running when any api or connection error occured.
How should I handle this error ?
while True:
..............
try :
klines = client.get_historical_klines(ticker_symbol, Client.KLINE_INTERVAL_1MINUTE, _from_str, _until_str)
except BinanceAPIException as e:
print(e)
print('Something went wrong. Error occured at %s. Wait for 1 hour.' % (datetime.datetime.now().astimezone(timezone('UTC'))))
sleep(3600)
continue
..............
All error massege are following.
Traceback (most recent call last):
File
"/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/site-packages/urllib3/connectionpool.py",
line 600, in urlopen
chunked=chunked) File "/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/site-packages/urllib3/connectionpool.py",
line 384, in _make_request
six.raise_from(e, None) File "", line 2, in raise_from File
"/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/site-packages/urllib3/connectionpool.py",
line 380, in _make_request
httplib_response = conn.getresponse() File "/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/http/client.py",
line 1321, in getresponse
response.begin() File "/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/http/client.py",
line 296, in begin
version, status, reason = self._read_status() File "/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/http/client.py",
line 257, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") File "/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/socket.py", line
589, in readinto
return self._sock.recv_into(b) File "/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/site-packages/urllib3/contrib/pyopenssl.py",
line 312, in recv_into
return self.recv_into(*args, **kwargs) File "/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/site-packages/urllib3/contrib/pyopenssl.py",
line 302, in recv_into
raise SocketError(str(e)) OSError: (104, 'ECONNRESET')
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File
"/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/site-packages/requests/adapters.py",
line 449, in send
timeout=timeout File "/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/site-packages/urllib3/connectionpool.py",
line 638, in urlopen
_stacktrace=sys.exc_info()[2]) File "/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/site-packages/urllib3/util/retry.py",
line 368, in increment
raise six.reraise(type(error), error, _stacktrace) File "/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/site-packages/urllib3/packages/six.py",
line 685, in reraise
raise value.with_traceback(tb) File "/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/site-packages/urllib3/connectionpool.py",
line 600, in urlopen
chunked=chunked) File "/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/site-packages/urllib3/connectionpool.py",
line 384, in _make_request
six.raise_from(e, None) File "", line 2, in raise_from File
"/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/site-packages/urllib3/connectionpool.py",
line 380, in _make_request
httplib_response = conn.getresponse() File "/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/http/client.py",
line 1321, in getresponse
response.begin() File "/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/http/client.py",
line 296, in begin
version, status, reason = self._read_status() File "/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/http/client.py",
line 257, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") File "/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/socket.py", line
589, in readinto
return self._sock.recv_into(b) File "/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/site-packages/urllib3/contrib/pyopenssl.py",
line 312, in recv_into
return self.recv_into(*args, **kwargs) File "/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/site-packages/urllib3/contrib/pyopenssl.py",
line 302, in recv_into
raise SocketError(str(e)) urllib3.exceptions.ProtocolError: ('Connection aborted.', OSError("(104, 'ECONNRESET')"))
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File
"BTC_USDT_BINANCE_minutes.py", line 54, in
klines = client.get_historical_klines(ticker_symbol, Client.KLINE_INTERVAL_1MINUTE, _from_str, _until_str) File
"/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/site-packages/binance/client.py",
line 765, in get_historical_klines
first_valid_ts = self._get_earliest_valid_timestamp(symbol, interval) File
"/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/site-packages/binance/client.py",
line 723, in _get_earliest_valid_timestamp
endTime=None File "/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/site-packages/binance/client.py",
line 705, in get_klines
return self._get('klines', data=params) File "/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/site-packages/binance/client.py",
line 207, in _get
return self._request_api('get', path, signed, version, **kwargs) File
"/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/site-packages/binance/client.py",
line 181, in _request_api
return self._request(method, uri, signed, **kwargs) File "/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/site-packages/binance/client.py",
line 175, in _request
response = getattr(self.session, method)(uri, **kwargs) File "/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/site-packages/requests/sessions.py",
line 546, in get
return self.request('GET', url, **kwargs) File "/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/site-packages/requests/sessions.py",
line 533, in request
resp = self.send(prep, **send_kwargs) File "/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/site-packages/requests/sessions.py",
line 646, in send
r = adapter.send(request, **kwargs) File "/home/ec2-user/.pyenv/versions/3.7.3/lib/python3.7/site-packages/requests/adapters.py",
line 498, in send
raise ConnectionError(err, request=request) requests.exceptions.ConnectionError: ('Connection aborted.',
OSError("(104, 'ECONNRESET')"))
Assuming that you are using python-binance module, most likely the error happens because you are trying to reuse the client after long time (server timeout). You can try to recreate the client first:
...
client = Client(api_key, api_secret)
while True:
...
try :
klines = client.get_historical_klines(ticker_symbol, Client.KLINE_INTERVAL_1MINUTE, _from_str, _until_str)
except BinanceAPIException as e:
print(e)
print('Something went wrong. Error occured at %s. Wait for 1 hour.' % (datetime.datetime.now().astimezone(timezone('UTC'))))
sleep(3600)
client = Client(api_key, api_secret)
continue
...
The reason is because the [Client][2] uses requests Session underneath which imply usage of urllib3's connection pooling. This connection pooling can not be reused after timeout.
I am writing a simply python script for vkontakte.com (vk.com). It's main task is to run constantly in background, tracking one exact group chat. When it comes to a new message in chat, script answers back on special keywords. Nothing hard. I am kinda new to python and it's connection architecture. After a random time script fails due ConnectionResetError when it comes to answer back with vk.messages.send() function. Fails completly random - might work for 9 minutes or 20 minutes, most of time fails during message send, but also fails if i leave script working for 20+ minutes time. What do i do now ? I need it to work constantly. Also i want to understand why connection fails.
main loop that tracks new messages is written like this:
for event in longpoll.listen():
if event.type == VkEventType.MESSAGE_NEW and event.to_me and event.text:
traceback of error:
Traceback (most recent call last):
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\site-packages\urllib3\connectionpool.py", line 600, in urlopen
chunked=chunked)
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\site-packages\urllib3\connectionpool.py", line 384, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\site-packages\urllib3\connectionpool.py", line 380, in _make_request
httplib_response = conn.getresponse()
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\http\client.py", line 1321, in getresponse
response.begin()
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\http\client.py", line 296, in begin
version, status, reason = self._read_status()
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\http\client.py", line 257, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\socket.py", line 589, in readinto
return self._sock.recv_into(b)
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\ssl.py", line 1052, in recv_into
return self.read(nbytes, buffer)
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\ssl.py", line 911, in read 17:58
return self._sslobj.read(len, buffer)
ConnectionResetError: [WinError 10054] Удаленный хост принудительно разорвал существующее подключение (remote peer forcing a connection to close)
continue of traceback:
Traceback (most recent call last):
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\site-packages\requests\adapters.py", line 449, in send
timeout=timeout
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\site-packages\urllib3\connectionpool.py", line 638, in urlopen
_stacktrace=sys.exc_info()[2])
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\site-packages\urllib3\util\retry.py", line 367, in increment
raise six.reraise(type(error), error, _stacktrace)
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\site-packages\urllib3\packages\six.py", line 685, in reraise
raise value.with_traceback(tb)
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\site-packages\urllib3\connectionpool.py", line 600, in urlopen
chunked=chunked)
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\site-packages\urllib3\connectionpool.py", line 384, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\site-packages\urllib3\connectionpool.py", line 380, in _make_request
httplib_response = conn.getresponse()
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\http\client.py", line 1321, in getresponse
response.begin()
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\http\client.py", line 296, in begin
version, status, reason = self._read_status()
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\http\client.py", line 257, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\socket.py", line 589, in readinto
return self._sock.recv_into(b)
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\ssl.py", line 1052, in recv_into
return self.read(nbytes, buffer)
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\ssl.py", line 911, in read
return self._sslobj.read(len, buffer)
urllib3.exceptions.ProtocolError: ('Connection aborted.', ConnectionResetError(10054, 'Удаленный хост принудительно разорвал существующее подключение', None, 10054, None))
and last one:
Traceback (most recent call last):
File "vsk_bot.py", line 37, in <module>
vk.messages.send(
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\site-packages\vk_api\vk_api.py", line 671, in __call__
return self._vk.method(self._method, kwargs)
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\site-packages\vk_api\vk_api.py", line 602, in method
values
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\site-packages\requests\sessions.py", line 581, in post
return self.request('POST', url, data=data, json=json, **kwargs)
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\site-packages\requests\sessions.py", line 533, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\site-packages\requests\sessions.py", line 646, in send
r = adapter.send(request, **kwargs)
File "C:\Users\XXX\AppData\Local\Programs\Python\Python37-32\lib\site-packages\requests\adapters.py", line 498, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', ConnectionResetError(10054, 'Удаленный хост принудительно разорвал существующее подключение', None, 10054, None))