I'm scraping twitter trying to get the friends/users being followed for a list of twitter users. I'm using tweepy and python 3.6.5 on OSX 10.13. An abbreviated code chunk :
def get_friends_for_each_twitter_user(UserL=None, Name=None):
.
. # Auth keys and such
.
for user in UserL: ### This is a list of USER class with the below fields ###
### Handle protected users ###
if(user.protected == True):
user.friendsL = "protected"
continue
screenNameL=[]
friendIDL=[]
friendL=[]
friendScreenNameL=[]
### Get IDs of people that this user follows (i.e. 'friends') ###
for page in tweepy.Cursor(api.friends_ids, screen_name=user.screenName).pages():
friendIDL.extend(page)
time.sleep(60)
## Loop through IDs, get user profile, keep only friends' screen name ###
for i in range(0, len(friendIDL), 100):
friendL.extend(api.lookup_users(user_ids=friendIDL[i:i+100]))
### Keep only screen name ###
for friend in friendL:
friendScreenNameL.append(friend._json['screen_name'])
user.friendsL = friendScreenNameL
When I do this, after collecting the friends (i.e. profiles that the user follows) for about a dozen users, I get the following errors:
Traceback (most recent call last):
File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/urllib3/connectionpool.py", line 601, in urlopen
chunked=chunked)
File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/urllib3/connectionpool.py", line 387, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/urllib3/connectionpool.py", line 383, in _make_request
httplib_response = conn.getresponse()
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1331, in getresponse
response.begin()
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 297, in begin
version, status, reason = self._read_status()
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 266, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/requests/adapters.py", line 440, in send
timeout=timeout
File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/urllib3/connectionpool.py", line 639, in urlopen
_stacktrace=sys.exc_info()[2])
File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/urllib3/util/retry.py", line 357, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/urllib3/packages/six.py", line 685, in reraise
raise value.with_traceback(tb)
File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/urllib3/connectionpool.py", line 601, in urlopen
chunked=chunked)
File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/urllib3/connectionpool.py", line 387, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/urllib3/connectionpool.py", line 383, in _make_request
httplib_response = conn.getresponse()
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1331, in getresponse
response.begin()
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 297, in begin
version, status, reason = self._read_status()
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 266, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/tweepy/binder.py", line 190, in execute
proxies=self.api.proxy)
File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/requests/sessions.py", line 508, in request
resp = self.send(prep, **send_kwargs)
File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/requests/sessions.py", line 618, in send
r = adapter.send(request, **kwargs)
File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/requests/adapters.py", line 490, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pdb.py", line 1667, in main
pdb._runscript(mainpyfile)
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pdb.py", line 1548, in _runscript
self.run(statement)
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/bdb.py", line 434, in run
exec(cmd, globals, locals)
File "<string>", line 1, in <module>
File "/Users/myusername/Code/Python/hair_prod/src/main.py", line 170, in <module>
main()
File "/Users/myusername/Code/Python/hair_prod/src/main.py", line 141, in main
get_friends_for_each_twitter_user(UserL=tresemmeUserL, Name="Tresemme")
File "src/twitter_scraper.py", line 187, in get_friends_for_each_twitter_user
friendL.extend(api.lookup_users(user_ids=friendIDL[i:i+100]))
File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/tweepy/api.py", line 336, in lookup_users
return self._lookup_users(post_data=post_data)
File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/tweepy/binder.py", line 250, in _call
return method.execute()
File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/tweepy/binder.py", line 192, in execute
six.reraise(TweepError, TweepError('Failed to send request: %s' % e), sys.exc_info()[2])
File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/six.py", line 692, in reraise
raise value.with_traceback(tb)
File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/tweepy/binder.py", line 190, in execute
proxies=self.api.proxy)
File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/requests/sessions.py", line 508, in request
resp = self.send(prep, **send_kwargs)
File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/requests/sessions.py", line 618, in send
r = adapter.send(request, **kwargs)
File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/requests/adapters.py", line 490, in send
raise ConnectionError(err, request=request)
tweepy.error.TweepError: Failed to send request: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))
Uncaught exception. Entering post mortem debugging
Running 'cont' or 'step' will restart the program
> /Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/requests/adapters.py(490)send()
-> raise ConnectionError(err, request=request)
It appears that the errors occur on line friendL.extend(api.lookup_users(user_ids=friendIDL[i:i+100])) in the
get_friends_for_each_twitter_user() function
QUESTION :
Why is this error occurring?
How do I avoid/work around it?
Any number of things could cause the error to appear, but if the cause is not permanent, then retrying an occasional failed API call could make the script work alright.
According to the Tweepy docs the API client constructor accepts a retry_count parameter which defaults to 0. Try setting retry_count to something above 0 and see if your script is able to complete successfully, something like this:
api = tweepy.api.API(..., retry_count=3)
After messing with this for a while, I believe that this issue was caused by my network connection. When this happened, I was connected to my 5GHz wireless network. When I connected to my 2.4GHz wireless network, these errors are less frequent. The proper thing to do in this situation is to handle the exception, wait a few seconds and try again. Below is the appropriate code fragment:
def get_friends_for_each_twitter_user(UserL=None, Name=None):
consumerKey = #your value here
consumerSecret = #your value here
auth = tweepy.AppAuthHandler(consumerKey, consumerSecret) ### Supposedly faster
api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True) ## Now I don't have to handle rate limiting myself
for user in UserL:
accountStatus = 'active'
if(user.protected == True):
user.friendsL = "protected"
continue
screenNameL=[]
friendIDL=[]
friendL=[]
friendScreenNameL=[]
#### TWITTER LIMITS US #####
try :
for page in tweepy.Cursor(api.friends_ids, screen_name=user.screenName).pages():
friendIDL.extend(page)
except tweepy.TweepError as error :
if(error.__dict__['api_code'] == 34):
accountStatus = 'dead'
print("...{} is dead".format(user.screenName))
continue
else:
raise
for i in range(0, len(friendIDL), 100):
### This handles when exception occurs (probably due to connection issues)
### When exception occurs, sleeps then retries. I don't notice this error
### when I'm running on corporate Wifi, maybe my router just sucks
while True:
try :
friendL.extend(api.lookup_users(user_ids=friendIDL[i:i+100]))
except tweepy.TweepError as error :
print("...Exception for {} : api_code {}".format(user.screenName,
error.__dict__['api_code']))
time.sleep(5)
continue
break
Related
I have a python script in a docker that runs and collects data from an api on a loop.
`for idx, items in enumerate(SearchList):
#get access token
access_token = GetPricingAccTok()
#request id to get results
reqid = RequestIdCompSearch(items, access_token)
#api to get results by doing a while loop appending results together till api returns 100% as completed percentage
Result = GetReqResults(reqid, access_token)
Pricings = GetPricingDict(Result)
filenamepricing = "pricinghistory/CorrectFormat/Pricings_Data_{idn}_{ts}.json".format(
idn = idx,
ts= dt.datetime.now().isoformat()
)
GetAzContainer().upload_blob(
name=filenamepricing,
data=json.dumps(obj= Pricings, indent=4),
blob_type='BlockBlob'
)
`
When i run and test this code locally on my windows machine it works well without an issue and completes the requests
but in the docker on azure container instance it then returns an error showing that the connection has timed out. Not sure what the difference is between the running it locally and in the docker that i then recieve this error.
I have tried addinng this code at the start
import socket
from urllib3.connection import HTTPConnection
HTTPConnection.default_socket_options =(HTTPConnection.default_socket_options + [
(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) #Enables the feature
,(socket.SOL_TCP, socket.TCP_KEEPIDLE, 45) #Overrides the time when the stack willl start sending KeppAlives after no data received on a Persistent Connection
,(socket.SOL_TCP, socket.TCP_KEEPINTVL, 10) #Defines how often thoe KA will be sent between them
,(socket.SOL_TCP, socket.TCP_KEEPCNT, 60) #How many attemps will your code try if the server goes down before droping the connection.
]
)
but it still times out. showing the below error
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 445, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 440, in _make_request
httplib_response = conn.getresponse()
File "/usr/local/lib/python3.8/http/client.py", line 1348, in getresponse
response.begin()
File "/usr/local/lib/python3.8/http/client.py", line 316, in begin
version, status, reason = self._read_status()
File "/usr/local/lib/python3.8/http/client.py", line 277, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/local/lib/python3.8/socket.py", line 669, in readinto
return self._sock.recv_into(b)
File "/usr/local/lib/python3.8/ssl.py", line 1241, in recv_into
return self.read(nbytes, buffer)
File "/usr/local/lib/python3.8/ssl.py", line 1099, in read
return self._sslobj.read(len, buffer)
TimeoutError: [Errno 110] Connection timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/requests/adapters.py", line 440, in send
resp = conn.urlopen(
File "/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 755, in urlopen
retries = retries.increment(
File "/usr/local/lib/python3.8/site-packages/urllib3/util/retry.py", line 532, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/usr/local/lib/python3.8/site-packages/urllib3/packages/six.py", line 770, in reraise
raise value
File "/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 447, in _make_request
self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
File "/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 353, in _raise_timeout
raise ReadTimeoutError(
urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='service.xxx.com', port=443): Read timed out. (read timeout=None)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "src/script.py", line 298, in <module>
reqid = RequestIdCompSearch(items, access_token)
File "src/script.py", line 111, in RequestIdCompSearch
InitResp =rq.post( ReqUrl, headers= ReqHead, data=Jsondata)
File "/usr/local/lib/python3.8/site-packages/requests/api.py", line 117, in post
return request('post', url, data=data, json=json, **kwargs)
File "/usr/local/lib/python3.8/site-packages/requests/api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/local/lib/python3.8/site-packages/requests/sessions.py", line 529, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python3.8/site-packages/requests/sessions.py", line 645, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python3.8/site-packages/requests/adapters.py", line 532, in send
raise ReadTimeout(e, request=request)
requests.exceptions.ReadTimeout: HTTPSConnectionPool(host='service.xxxx.com', port=443): Read timed out. (read timeout=None)
I have two telegram bots made with PyTelegramBotApi (not the best library, I know) deployed on Heroku. One big difference that matters: one is running with Threading to periodically send notifications to "subscribers" (around 20 people). And it crashes approximately every month with following exception trace in Heroku console:
2022-01-14T08:00:11.068553+00:00 app[worker.1]: File "/app/.heroku/python/lib/python3.9/site-packages/telebot/apihelper.py", line 139, in _make_request
2022-01-14T08:00:11.068739+00:00 app[worker.1]: result = _get_req_session().request(
2022-01-14T08:00:11.068748+00:00 app[worker.1]: File "/app/.heroku/python/lib/python3.9/site-packages/requests/sessions.py", line 542, in request
2022-01-14T08:00:11.068983+00:00 app[worker.1]: resp = self.send(prep, **send_kwargs)
2022-01-14T08:00:11.068994+00:00 app[worker.1]: File "/app/.heroku/python/lib/python3.9/site-packages/requests/sessions.py", line 655, in send
2022-01-14T08:00:11.069233+00:00 app[worker.1]: r = adapter.send(request, **kwargs)
2022-01-14T08:00:11.069242+00:00 app[worker.1]: File "/app/.heroku/python/lib/python3.9/site-packages/requests/adapters.py", line 529, in send
2022-01-14T08:00:11.069448+00:00 app[worker.1]: raise ReadTimeout(e, request=request)
2022-01-14T08:00:11.069478+00:00 app[worker.1]: requests.exceptions.ReadTimeout: HTTPSConnectionPool(host='api.telegram.org', port=443): Read timed out. (read timeout=25)
I didn't record previous exception traces (my bad), but I can remember it looking pretty much the same (with the most recent exception being ReadTimeout). Moreover, any of my scripts were never seen in this trace (and this makes me feel it's incomplete due to some Heroku log console bug). I myself don't use requests module directly. Also it definitely can't be caused by dyno sleep as I use only worker dyno (it never sleeps). Both this bot and the one not crashing are using Heroku Postgres.
Any ideas on what could be causing this exception?
EDIT:
full exception trace I accidentally got in debug mode while testing somethin else on my machine:
Traceback (most recent call last):
File "D:\Deadliner0307\lib\site-packages\urllib3\connectionpool.py", line 445, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "D:\Deadliner0307\lib\site-packages\urllib3\connectionpool.py", line 440, in _make_request
httplib_response = conn.getresponse()
File "C:\Users\vva07\AppData\Local\Programs\Python\Python39-32\lib\http\client.py", line 1347, in getresponse
response.begin()
File "C:\Users\vva07\AppData\Local\Programs\Python\Python39-32\lib\http\client.py", line 307, in begin
version, status, reason = self._read_status()
File "C:\Users\vva07\AppData\Local\Programs\Python\Python39-32\lib\http\client.py", line 268, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "C:\Users\vva07\AppData\Local\Programs\Python\Python39-32\lib\socket.py", line 704, in readinto
return self._sock.recv_into(b)
File "C:\Users\vva07\AppData\Local\Programs\Python\Python39-32\lib\ssl.py", line 1241, in recv_into
return self.read(nbytes, buffer)
File "C:\Users\vva07\AppData\Local\Programs\Python\Python39-32\lib\ssl.py", line 1099, in read
return self._sslobj.read(len, buffer)
socket.timeout: The read operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "D:\Deadliner0307\lib\site-packages\requests\adapters.py", line 439, in send
resp = conn.urlopen(
File "D:\Deadliner0307\lib\site-packages\urllib3\connectionpool.py", line 755, in urlopen
retries = retries.increment(
File "D:\Deadliner0307\lib\site-packages\urllib3\util\retry.py", line 532, in increment
raise six.reraise(type(error), error, _stacktrace)
File "D:\Deadliner0307\lib\site-packages\urllib3\packages\six.py", line 770, in reraise
raise value
File "D:\Deadliner0307\lib\site-packages\urllib3\connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "D:\Deadliner0307\lib\site-packages\urllib3\connectionpool.py", line 447, in _make_request
self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
File "D:\Deadliner0307\lib\site-packages\urllib3\connectionpool.py", line 336, in _raise_timeout
raise ReadTimeoutError(
urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='api.telegram.org', port=443): Read timed out. (read timeout=25)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\vva07\OneDrive\Документы\проекты\Deadliner0307\main.py", line 450, in <module>
bot.polling(none_stop=True, interval=1)
File "C:\Users\vva07\OneDrive\Документы\проекты\Deadliner0307\main.py", line 446, in deadliner0307
else:
File "D:\Deadliner0307\lib\site-packages\telebot\__init__.py", line 633, in polling
self.__threaded_polling(non_stop, interval, timeout, long_polling_timeout, allowed_updates)
File "D:\Deadliner0307\lib\site-packages\telebot\__init__.py", line 692, in __threaded_polling
raise e
File "D:\Deadliner0307\lib\site-packages\telebot\__init__.py", line 654, in __threaded_polling
polling_thread.raise_exceptions()
File "D:\Deadliner0307\lib\site-packages\telebot\util.py", line 100, in raise_exceptions
raise self.exception_info
File "D:\Deadliner0307\lib\site-packages\telebot\util.py", line 82, in run
task(*args, **kwargs)
File "D:\Deadliner0307\lib\site-packages\telebot\__init__.py", line 391, in __retrieve_updates
updates = self.get_updates(offset=(self.last_update_id + 1),
File "D:\Deadliner0307\lib\site-packages\telebot\__init__.py", line 371, in get_updates
json_updates = apihelper.get_updates(self.token, offset, limit, timeout, allowed_updates, long_polling_timeout)
File "D:\Deadliner0307\lib\site-packages\telebot\apihelper.py", line 312, in get_updates
return _make_request(token, method_url, params=payload)
File "D:\Deadliner0307\lib\site-packages\telebot\apihelper.py", line 139, in _make_request
result = _get_req_session().request(
File "D:\Deadliner0307\lib\site-packages\requests\sessions.py", line 542, in request
resp = self.send(prep, **send_kwargs)
File "D:\Deadliner0307\lib\site-packages\requests\sessions.py", line 655, in send
r = adapter.send(request, **kwargs)
File "D:\Deadliner0307\lib\site-packages\requests\adapters.py", line 529, in send
raise ReadTimeout(e, request=request)
requests.exceptions.ReadTimeout: HTTPSConnectionPool(host='api.telegram.org', port=443): Read timed out. (read timeout=25)
I have found a way to at least keep the bot working. Be careful: this may not fit your application (in your case restarting straight up after a specific exception may corrupt data or break something else). In main file, I made this construction, which basically restarts a bot after an unhandled exception, which would otherwise just stop the bot:
def exception_handler(count: int = 0):
"""Relaunching bot unless exceptions occur more than 2 times a day
(script is reset daily on Heroku)"""
if count < 3:
if count > 0:
print("An exception occurred, relaunching . . .")
time.sleep(5)
try:
deadliner0307() # bot main function with (bot.polling starts there)
except Exception as ex:
count += 1
# Notifying myself about exception via free Airbrake addon:
notifier = pybrake.Notifier(project_id=399289,
project_key='129d3450356965175fda762b69e1babf',
environment='production')
notifier.notify(ex)
exception_handler(count)
else:
print("Too much exceptions occurred, shutting down . . .")
if __name__ == '__main__':
exception_handler()
Below is my Python script in a file called P2_small.py, trimmed down and simplified to not make it wieldy. Basically it sets up an infinite loop each time calling yfinance to get data within a try/except construct. When the internet goes down, I get the error below. Two likely related questions:
How come that the stackdump does not mention my script (as you can see it refers to all sorts of .py files but never to P2_small.py)?
Why does this error not get trapped by my try/except construct?
Code in P2_small.py:
import requests
from tkinter import *
import yfinance as yf
def isInternet(): #Method to check there is a good internet connection
url = 'http://www.google.com/'
try:
_ = requests.get(url, timeout=3)
return True
except requests.ConnectionError:
return False
def startInfiniteLoop():
try:
if not isInternet():
print("No internet connection - waiting...");
else:
df = yf.download(tickers="RDSB.L", period="1y", interval="1d", auto_adjust=True, prepost=False)
if df.empty:
print("Does not exist")
else:
print(df.info)
# Call startInfiniteLoop() again and again and again, each time after 10 secs
root.after(10000, startInfiniteLoop)
except Exception as exception1:
print("Error1 occured: " + str(exception1))
#Make GUI
root = Tk()
root.geometry("300x3000")
#Start infinite loop and show the GUI
root.after(100, startInfiniteLoop)
root.mainloop()
Errors generated:
Exception in thread Thread-1225:
Traceback (most recent call last):
File "C:\Users\mkemp\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connection.py", line 169, in _new_conn
conn = connection.create_connection(
File "C:\Users\mkemp\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\util\connection.py", line 96, in create_connection
raise err
File "C:\Users\mkemp\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\util\connection.py", line 86, in create_connection
sock.connect(sa)
OSError: [WinError 10065] A socket operation was attempted to an unreachable host
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\mkemp\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "C:\Users\mkemp\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 382, in _make_request
self._validate_conn(conn)
File "C:\Users\mkemp\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 1010, in _validate_conn
conn.connect()
File "C:\Users\mkemp\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connection.py", line 353, in connect
conn = self._new_conn()
File "C:\Users\mkemp\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connection.py", line 181, in _new_conn
raise NewConnectionError(
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPSConnection object at 0x000002518AA6B100>: Failed to establish a new connection: [WinError 10065] A socket operation was attempted to an unreachable host
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\mkemp\AppData\Local\Programs\Python\Python39\lib\site-packages\requests\adapters.py", line 439, in send
resp = conn.urlopen(
File "C:\Users\mkemp\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 755, in urlopen
retries = retries.increment(
File "C:\Users\mkemp\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\util\retry.py", line 574, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='query2.finance.yahoo.com', port=443): Max retries exceeded with url: /v8/finance/chart/SAF.PA?range=0d&interval=1d&includePrePost=False&events=div%2Csplits (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x000002518AA6B100>: Failed to establish a new connection: [WinError 10065] A socket operation was attempted to an unreachable host'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\mkemp\AppData\Local\Programs\Python\Python39\lib\threading.py", line 954, in _bootstrap_inner
self.run()
File "C:\Users\mkemp\AppData\Local\Programs\Python\Python39\lib\threading.py", line 892, in run
self._target(*self._args, **self._kwargs)
File "C:\Users\mkemp\AppData\Local\Programs\Python\Python39\lib\site-packages\multitasking\__init__.py", line 102, in _run_via_pool
return callee(*args, **kwargs)
File "C:\Users\mkemp\AppData\Local\Programs\Python\Python39\lib\site-packages\yfinance\multi.py", line 169, in _download_one_threaded
data = _download_one(ticker, start, end, auto_adjust, back_adjust,
File "C:\Users\mkemp\AppData\Local\Programs\Python\Python39\lib\site-packages\yfinance\multi.py", line 181, in _download_one
return Ticker(ticker).history(period=period, interval=interval,
File "C:\Users\mkemp\AppData\Local\Programs\Python\Python39\lib\site-packages\yfinance\base.py", line 152, in history
data = self.session.get(
File "C:\Users\mkemp\AppData\Local\Programs\Python\Python39\lib\site-packages\requests\api.py", line 76, in get
return request('get', url, params=params, **kwargs)
File "C:\Users\mkemp\AppData\Local\Programs\Python\Python39\lib\site-packages\requests\api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Users\mkemp\AppData\Local\Programs\Python\Python39\lib\site-packages\requests\sessions.py", line 542, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\mkemp\AppData\Local\Programs\Python\Python39\lib\site-packages\requests\sessions.py", line 655, in send
r = adapter.send(request, **kwargs)
File "C:\Users\mkemp\AppData\Local\Programs\Python\Python39\lib\site-packages\requests\adapters.py", line 516, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPSConnectionPool(host='query2.finance.yahoo.com', port=443): Max retries exceeded with url: /v8/finance/chart/SAF.PA?range=0d&interval=1d&includePrePost=False&events=d
I'm having this nasty error:
Traceback (most recent call last):
File "/home/ubuntu/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/home/ubuntu/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 384, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "/home/ubuntu/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 380, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.5/http/client.py", line 1197, in getresponse
response.begin()
File "/usr/lib/python3.5/http/client.py", line 297, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.5/http/client.py", line 258, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/lib/python3.5/socket.py", line 575, in readinto
return self._sock.recv_into(b)
ConnectionResetError: [Errno 104] Connection reset by peer
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/ubuntu/.local/lib/python3.5/site-packages/requests/adapters.py", line 445, in send
timeout=timeout
File "/home/ubuntu/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 638, in urlopen
_stacktrace=sys.exc_info()[2])
File "/home/ubuntu/.local/lib/python3.5/site-packages/urllib3/util/retry.py", line 367, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/home/ubuntu/.local/lib/python3.5/site-packages/urllib3/packages/six.py", line 685, in reraise
raise value.with_traceback(tb)
File "/home/ubuntu/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/home/ubuntu/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 384, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "/home/ubuntu/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 380, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.5/http/client.py", line 1197, in getresponse
response.begin()
File "/usr/lib/python3.5/http/client.py", line 297, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.5/http/client.py", line 258, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/lib/python3.5/socket.py", line 575, in readinto
return self._sock.recv_into(b)
urllib3.exceptions.ProtocolError: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/ubuntu/script.py", line 44, in <module>
response_service_k = service_k_lib.service_kData(data, access_token, apifolder)
File "/home/ubuntu/script_lib.py", line 238, in service_kData
datarALL.extend(data.result())
File "/usr/lib/python3.5/concurrent/futures/_base.py", line 398, in result
return self.__get_result()
File "/usr/lib/python3.5/concurrent/futures/_base.py", line 357, in __get_result
raise self._exception
File "/usr/lib/python3.5/concurrent/futures/thread.py", line 55, in run
result = self.fn(*self.args, **self.kwargs)
File "/home/ubuntu/script_lib.py", line 129, in getdata3
responsedata = requests.get(url, data=data, headers=hed, verify=False)
File "/home/ubuntu/.local/lib/python3.5/site-packages/requests/api.py", line 72, in get
return request('get', url, params=params, **kwargs)
File "/home/ubuntu/.local/lib/python3.5/site-packages/requests/api.py", line 58, in request
return session.request(method=method, url=url, **kwargs)
File "/home/ubuntu/.local/lib/python3.5/site-packages/requests/sessions.py", line 512, in request
resp = self.send(prep, **send_kwargs)
File "/home/ubuntu/.local/lib/python3.5/site-packages/requests/sessions.py", line 622, in send
r = adapter.send(request, **kwargs)
File "/home/ubuntu/.local/lib/python3.5/site-packages/requests/adapters.py", line 495, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer'))
According to this answer the solution for my issue is to place :
time.sleep(0.01) strategically in my code. I get the idea of it but I'm not sure I know where the correct spot for it in my code:
This is the relevant code part from script.py :
result= []
with ThreadPoolExecutor(max_workers=num_of_pages) as executor:
futh = [(executor.submit(self.getdata3, page, hed, data, apifolder,additional)) for page in pages]
for data in as_completed(futh):
result.extend(data.result())
print ("Finished generateing data.")
return result
This is the relevant code part from getdata3 function :
def getdata3(...)
datarALL = []
responsedata = requests.get(url, data=data, headers=hed, verify=False)
if responsedata.status_code == 200: # 200 for successful call
responsedata = responsedata.text
jsondata = json.loads(responsedata)
if "results" in jsondata:
if jsondata["results"]:
datarALL.extend(jsondata["results"])
print ("{1} page {0} finished".format(page,str(datetime.now())))
return datarALL
Some info:
My code generates pages.
Create thread per page executing getdata3 (which makes GET request to API)
each thread return the result of a single page.
"merging" results.
My question:
Where to put the time.sleep(0.01) in order to avoid this error?
you might already figured this out. Any way, I had a similar problem while making post requests in a for loop, which run 400 times, each loop run with two requests and a last request after exiting the loop. I found out while making API requests the server (check_mk) was not ready, still compiling some hots and service data. So I placed the time.sleep() just before the each request and that fixed the problem. I don't know how often you make the requests but I'd try it this way:
def getdata3(...)
datarALL = []
time.sleep(0.01)
responsedata = requests.get(url, data=data, headers=hed, verify=False)
if responsedata.status_code == 200: # 200 for successful call
responsedata = responsedata.text
jsondata = json.loads(responsedata)
if "results" in jsondata:
if jsondata["results"]:
datarALL.extend(jsondata["results"])
print ("{1} page {0} finished".format(page,str(datetime.now())))
return datarALL
As suggested in this thread by the same user posting the problem, it might be due to your internet connection. I had the same problem and changed my internet connection to another network and the issue was solved for me. Then you can give it a try changing your internet network.
I made simple HTTP Server using BaseHTTPRequestHandler. The problem is, that when I want to post some data using requests from client, I get ConnectionError. I did simple request from requests lib documentation. Also interesting thing is, that HTTP Server will receive data from client and print it to console. I don't understand how its possible.
Client:
def post_data():
"""Client method"""
json_data = {
'sender': 'User',
'receiver': 'MY_SERVER',
'message': 'Hello server! Sending some data.'}
data_headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
data_payload = json.dumps(json_data)
try:
post = requests.post('http://localhost:8080/post', data=data_payload,
headers=data_headers)
print(post.status_code)
except ConnectionError as e:
print("CONNECTION ERROR: ")
print(e)
HTTP Server:
def do_POST(self):
"""Server method"""
self.send_response(200)
print("Receiving new data ...")
content_length = int(self.headers['Content-Length'])
post_data = self.rfile.read(content_length)
print(post_data)
Server result:
C:\Users\mypc\Projects\PythonFiles\httpserver>python server.py
Fri Jan 5 01:09:12 2018: HTTP Server started on port 8080.
127.0.0.1 - - [05/Jan/2018 01:09:21] "POST /post HTTP/1.1" 200 -
Receiving new data ...
b'{"sender": "User", "receiver": "MY_SERVER", "message": "Hello server! Sending some data."}'
Client result:
C:\Users\mypc\Projects\PythonFiles\httpserver>python client.py
CONNECTION ERROR:
('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))
C:\Users\mypc\Projects\PythonFiles\httpserver>
Error without exception block:
Traceback (most recent call last):
File "C:\Python36\lib\site-packages\urllib3\connectionpool.py", line 601, in urlopen
chunked=chunked)
File "C:\Python36\lib\site-packages\urllib3\connectionpool.py", line 387, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "C:\Python36\lib\site-packages\urllib3\connectionpool.py", line 383, in _make_request
httplib_response = conn.getresponse()
File "C:\Python36\lib\http\client.py", line 1331, in getresponse
response.begin()
File "C:\Python36\lib\http\client.py", line 297, in begin
version, status, reason = self._read_status()
File "C:\Python36\lib\http\client.py", line 266, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Python36\lib\site-packages\requests\adapters.py", line 440, in send
timeout=timeout
File "C:\Python36\lib\site-packages\urllib3\connectionpool.py", line 639, in urlopen
_stacktrace=sys.exc_info()[2])
File "C:\Python36\lib\site-packages\urllib3\util\retry.py", line 357, in increment
raise six.reraise(type(error), error, _stacktrace)
File "C:\Python36\lib\site-packages\urllib3\packages\six.py", line 685, in reraise
raise value.with_traceback(tb)
File "C:\Python36\lib\site-packages\urllib3\connectionpool.py", line 601, in urlopen
chunked=chunked)
File "C:\Python36\lib\site-packages\urllib3\connectionpool.py", line 387, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "C:\Python36\lib\site-packages\urllib3\connectionpool.py", line 383, in _make_request
httplib_response = conn.getresponse()
File "C:\Python36\lib\http\client.py", line 1331, in getresponse
response.begin()
File "C:\Python36\lib\http\client.py", line 297, in begin
version, status, reason = self._read_status()
File "C:\Python36\lib\http\client.py", line 266, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "client.py", line 137, in <module>
start_parser()
File "client.py", line 101, in start_parser
send_requests(args.get, args.post)
File "client.py", line 51, in send_requests
post_data()
File "client.py", line 129, in post_data
headers=data_headers)
File "C:\Python36\lib\site-packages\requests\api.py", line 112, in post
return request('post', url, data=data, json=json, **kwargs)
File "C:\Python36\lib\site-packages\requests\api.py", line 58, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Python36\lib\site-packages\requests\sessions.py", line 508, in request
resp = self.send(prep, **send_kwargs)
File "C:\Python36\lib\site-packages\requests\sessions.py", line 618, in send
r = adapter.send(request, **kwargs)
File "C:\Python36\lib\site-packages\requests\adapters.py", line 490, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))
It looks like the server is terminating the connection early without sending a full response. I've skimmed the docs and I think this is the problem (emphasis added):
send_response(code, message=None)
Adds a response header to the
headers buffer and logs the accepted request. The HTTP response line
is written to the internal buffer, followed by Server and Date
headers. The values for these two headers are picked up from the
version_string() and date_time_string() methods, respectively. If the
server does not intend to send any other headers using the
send_header() method, then send_response() should be followed by an
end_headers() call.
Changed in version 3.3: Headers are stored to an internal buffer and
end_headers() needs to be called explicitly.
So you probably just need to add the call to end_headers. If you were reading an old example (prior to Python 3.3) this wouldn't have been needed.
You can just put end_headers as Peter Gibson said
self.send_header('Content-Type', 'blabla' )
self.end_headers()