Owasp Zap module in python not working properly - python

I have created a script to perform an spider scan using owasp zap module from python. However, when start the script I get the following errors:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 670, in urlopen
httplib_response = self._make_request(
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 426, in _`enter code here`make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 421, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.9/http/client.py", line 1347, in getresponse
response.begin()
File "/usr/lib/python3.9/http/client.py", line 307, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.9/http/client.py", line 276, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 439, in send
resp = conn.urlopen(
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 724, in urlopen
retries = retries.increment(
File "/usr/lib/python3/dist-packages/urllib3/util/retry.py", line 439, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='127.0.0.1', port=8080): Max retries exceeded with url: http://zap/JSON/spider/action/scan/?apikey=&url=http%3A%2F%2F10.0.2.23 (Caused by ProxyError('Cannot connect to proxy.', RemoteDisconnected('Remote end closed connection without response')))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/kali/Desktop/vuln_discovery/main.py", line 27, in <module>
spider_scan(url)
File "/home/kali/Desktop/vuln_discovery/zap.py", line 12, in spider_scan
zap.spider.scan(url=url)
File "/home/kali/.local/lib/python3.9/site-packages/zapv2/spider.py", line 215, in scan
return six.next(six.itervalues(self.zap._request(self.zap.base + 'spider/action/scan/', params)))
File "/home/kali/.local/lib/python3.9/site-packages/zapv2/__init__.py", line 192, in _request
data = self._request_api(url, get)
File "/home/kali/.local/lib/python3.9/site-packages/zapv2/__init__.py", line 172, in _request_api
response = self.session.get(url, params=query, proxies=self.__proxies, verify=False)
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 543, in get
return self.request('GET', url, **kwargs)
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 530, in request
resp = self.send(prep, **send_kwargs)
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 643, in send
r = adapter.send(request, **kwargs)
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 510, in send
raise ProxyError(e, request=request)
enter code here
requests.exceptions.ProxyError: HTTPConnectionPool(host='127.0.0.1', port=8080): Max retries exceeded with url: http://zap/JSON/spider/action/scan/?apikey=&url=http%3A%2F%2F10.0.2.23 (Caused by ProxyError('Cannot connect to proxy.', RemoteDisconnected('Remote end closed connection without response')))`

I am not sure if that helps but I just had a similar problem with similar errors. For me the Port was not set properly. Maybe you can check the port with
sudo netstat -tunlp if the port there is really

Related

Failed to generate list of vault tokens

Looking for a solution to the problem to identify the vault root tokens, I found the following documentation
https://www.greenreedtech.com/identifying-active-hashicorp-vault-root-tokens/
The script related in this link worked in a vault instance that I have in version 1.5.4, but i have a problem listing the vault tokens in version 1.6.1
Traceback (most recent call last):
File "/home/vault/tokenlist-prod.py", line 31, in <module>
output = client.lookup_token(key, accessor=True)
File "/usr/local/lib/python3.10/dist-packages/hvac/v1/__init__.py", line 444, in lookup_token
return self._adapter.post(path, json=accessor_param, wrap_ttl=wrap_ttl)
File "/usr/local/lib/python3.10/dist-packages/hvac/adapters.py", line 126, in post
return self.request("post", url, **kwargs)
File "/usr/local/lib/python3.10/dist-packages/hvac/adapters.py", line 364, in request
response = super(JSONAdapter, self).request(*args, **kwargs)
File "/usr/local/lib/python3.10/dist-packages/hvac/adapters.py", line 330, in request
utils.raise_for_error(
File "/usr/local/lib/python3.10/dist-packages/hvac/utils.py", line 41, in raise_for_error
raise exceptions.Forbidden(message, errors=errors, method=method, url=url)
hvac.exceptions.Forbidden: 1 error occurred:
* permission denied
, on post https://mydomain/v1/auth/token/lookup-accessor
python3 tokenlist-alter.py
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 174, in _new_conn
conn = connection.create_connection(
File "/usr/lib/python3/dist-packages/urllib3/util/connection.py", line 96, in create_connection
raise err
File "/usr/lib/python3/dist-packages/urllib3/util/connection.py", line 86, in create_connection
sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 704, in urlopen
httplib_response = self._make_request(
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 399, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 239, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "/usr/lib/python3.10/http/client.py", line 1282, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.10/http/client.py", line 1328, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.10/http/client.py", line 1277, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.10/http/client.py", line 1037, in _send_output
self.send(msg)
File "/usr/lib/python3.10/http/client.py", line 975, in send
self.connect()
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 205, in connect
conn = self._new_conn()
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 186, in _new_conn
raise NewConnectionError(
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x7f398c14cd60>: Failed to establish a new connection: [Errno 111] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 440, in send
resp = conn.urlopen(
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 786, in urlopen
retries = retries.increment(
File "/usr/lib/python3/dist-packages/urllib3/util/retry.py", line 592, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='localhost', port=54453): Max retries exceeded with url: /v1/auth/token/accessors?list=True (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f398c14cd60>: Failed to establish a new connection: [Errno 111] Connection refused'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/vault/tokenlist-alter.py", line 25, in <module>
payload = client.list('auth/token/accessors')
File "/usr/local/lib/python3.10/dist-packages/hvac/v1/__init__.py", line 253, in list
return self._adapter.get("/v1/{0}".format(path), params=payload)
File "/usr/local/lib/python3.10/dist-packages/hvac/adapters.py", line 113, in get
return self.request("get", url, **kwargs)
File "/usr/local/lib/python3.10/dist-packages/hvac/adapters.py", line 364, in request
response = super(JSONAdapter, self).request(*args, **kwargs)
File "/usr/local/lib/python3.10/dist-packages/hvac/adapters.py", line 313, in request
response = self.session.request(
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 529, in request
resp = self.send(prep, **send_kwargs)
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 645, in send
r = adapter.send(request, **kwargs)
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 519, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='localhost', port=54453): Max retries exceeded with url: /v1/auth/token/accessors?list=True (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f398c14cd60>: Failed to establish a new connection: [Errno 111] Connection refused'))
The token I use to run the script is a root token
the urls i use have ssl certificate
Could someone help me with this problem?
Share my Script Script Py
Share Script
#!/usr/local/bin/python3
import os
import time
import hvac
import urllib3
from prettytable import PrettyTable
urllib3.disable_warnings()
try:
os.environ["VAULT_ADDR"] = "https://vault.test.com"
except Exception:
print("The VAULT_ADDR environment must be set.")
os._exit(1)
try:
os.environ["VAULT_TOKEN"] = "lalala"
except Exception:
print("The VAULT_TOKEN environment must be set.")
os._exit(1)
client = hvac.Client(url=os.environ['VAULT_ADDR'], verify=False, token=os.environ["VAULT_TOKEN"])
payload = client.list('auth/token/accessors')
keys = payload['data']['keys']
x = PrettyTable()
x.field_names = ["Display Name", "Creation Time", "Expiration Time", "Policies", "Token Accessor"]
for key in keys:
output = client.lookup_token(key, accessor=True)
display_name = output['data']['display_name']
creation_date = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(output['data']['creation_time']))
expire_time = output['data']['expire_time']
policies = output['data']['policies']
accessor = key
if "root" in policies:
x.add_row([display_name, creation_date, expire_time, policies, accessor])
print(x)

Got ssl.SSLEOFError when working behind a vpn

I m working behind a vpn and trying to use python requests, but I keep getting ssl.SSLEOFError: EOF occurred in violation of protocol (_ssl.c:1123). Once I turned down the vpn it worked as excepted. Is there any way to work around this issue?
Also one thing worth noting is that I tried to use postman while the vpn was on, and it worked fine.
Example
import requests
requests.get('https://www.google.com')
Traceback (most recent call last):
File "D:\Programming\Projects\Python\Scripts\venv\lib\site-packages\urllib3\connectionpool.py", line 700, in urlopen
self._prepare_proxy(conn)
File "D:\Programming\Projects\Python\Scripts\venv\lib\site-packages\urllib3\connectionpool.py", line 994, in _prepare_proxy
conn.connect()
File "D:\Programming\Projects\Python\Scripts\venv\lib\site-packages\urllib3\connection.py", line 364, in connect
conn = self._connect_tls_proxy(hostname, conn)
File "D:\Programming\Projects\Python\Scripts\venv\lib\site-packages\urllib3\connection.py", line 501, in _connect_tls_proxy
socket = ssl_wrap_socket(
File "D:\Programming\Projects\Python\Scripts\venv\lib\site-packages\urllib3\util\ssl_.py", line 453, in ssl_wrap_socket
ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls)
File "D:\Programming\Projects\Python\Scripts\venv\lib\site-packages\urllib3\util\ssl_.py", line 495, in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock)
File "D:\Programming\Python\Python39\lib\ssl.py", line 500, in wrap_socket
return self.sslsocket_class._create(
File "D:\Programming\Python\Python39\lib\ssl.py", line 1040, in _create
self.do_handshake()
File "D:\Programming\Python\Python39\lib\ssl.py", line 1309, in do_handshake
self._sslobj.do_handshake()
ssl.SSLEOFError: EOF occurred in violation of protocol (_ssl.c:1123)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "D:\Programming\Projects\Python\Scripts\venv\lib\site-packages\requests\adapters.py", line 440, in send
resp = conn.urlopen(
File "D:\Programming\Projects\Python\Scripts\venv\lib\site-packages\urllib3\connectionpool.py", line 785, in urlopen
retries = retries.increment(
File "D:\Programming\Projects\Python\Scripts\venv\lib\site-packages\urllib3\util\retry.py", line 592, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='www.google.com', port=443): Max retries exceeded with url: / (Caused by SSLError(SSLEOFError(8, 'EOF occurred in violation of protocol (_ssl.c:1123)')))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "D:\Programming\Projects\Python\Scripts\crawling\util.py", line 197, in <module>
main()
File "D:\Programming\Projects\Python\Scripts\crawling\util.py", line 189, in main
requests.get('https://www.google.com')
File "D:\Programming\Projects\Python\Scripts\venv\lib\site-packages\requests\api.py", line 75, in get
return request('get', url, params=params, **kwargs)
File "D:\Programming\Projects\Python\Scripts\venv\lib\site-packages\requests\api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "D:\Programming\Projects\Python\Scripts\venv\lib\site-packages\requests\sessions.py", line 529, in request
resp = self.send(prep, **send_kwargs)
File "D:\Programming\Projects\Python\Scripts\venv\lib\site-packages\requests\sessions.py", line 645, in send
r = adapter.send(request, **kwargs)
File "D:\Programming\Projects\Python\Scripts\venv\lib\site-packages\requests\adapters.py", line 517, in send
raise SSLError(e, request=request)
requests.exceptions.SSLError: HTTPSConnectionPool(host='www.google.com', port=443): Max retries exceeded with url: / (Caused by SSLError(SSLEOFError(8, 'EOF occurred in violation of protocol (_ssl.c:1123)')))
Thanks in advance.

pyTelegramBotAPI connection problem or what?

this is my code:
import telebot
bot = telebot.TeleBot(Token)
#bot.message_handler(commands=["greet"])
def greeting(message):
bot.reply_to(message, "Hi")
bot.polling()
and this is the error I receive:
Traceback (most recent call last):
File "C:\Users\pc\AppData\Local\Programs\Python\Python38-32\lib\site-packages\urllib3\connectionpool.py", line 696, in urlopen
self._prepare_proxy(conn)
File "C:\Users\pc\AppData\Local\Programs\Python\Python38-32\lib\site-packages\urllib3\connectionpool.py", line 964, in _prepare_proxy
conn.connect()
File "C:\Users\pc\AppData\Local\Programs\Python\Python38-32\lib\site-packages\urllib3\connection.py", line 359, in connect
conn = self._connect_tls_proxy(hostname, conn)
File "C:\Users\pc\AppData\Local\Programs\Python\Python38-32\lib\site-packages\urllib3\connection.py", line 496, in _connect_tls_proxy
return ssl_wrap_socket(
File "C:\Users\pc\AppData\Local\Programs\Python\Python38-32\lib\site-packages\urllib3\util\ssl_.py", line 432, in ssl_wrap_socket
ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls)
File "C:\Users\pc\AppData\Local\Programs\Python\Python38-32\lib\site-packages\urllib3\util\ssl_.py", line 474, in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock)
File "C:\Users\pc\AppData\Local\Programs\Python\Python38-32\lib\ssl.py", line 500, in wrap_socket
return self.sslsocket_class._create(
File "C:\Users\pc\AppData\Local\Programs\Python\Python38-32\lib\ssl.py", line 1040, in _create
self.do_handshake()
File "C:\Users\pc\AppData\Local\Programs\Python\Python38-32\lib\ssl.py", line 1309, in do_handshake
self._sslobj.do_handshake()
ssl.SSLError: [SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:1123)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\pc\AppData\Local\Programs\Python\Python38-32\lib\site-packages\requests\adapters.py", line 439, in send
resp = conn.urlopen(
File "C:\Users\pc\AppData\Local\Programs\Python\Python38-32\lib\site-packages\urllib3\connectionpool.py", line 755, in urlopen
retries = retries.increment(
File "C:\Users\pc\AppData\Local\Programs\Python\Python38-32\lib\site-packages\urllib3\util\retry.py", line 573, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='api.telegram.org', port=443): Max retries exceeded with url: /bot1494641875:AAEPX72rP7YFT0ZZzASh0cn9rAw6l8PUMk0/getUpdates?offset=1&timeout=20 (Caused by SSLError(SSLError(1, '[SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:1123)')))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "E:/Programming/Python/Training/TelegramBot02/main.py", line 13, in <module>
bot.polling()
File "C:\Users\pc\AppData\Local\Programs\Python\Python38-32\lib\site-packages\telebot\__init__.py", line 455, in polling
self.__threaded_polling(none_stop, interval, timeout, long_polling_timeout)
File "C:\Users\pc\AppData\Local\Programs\Python\Python38-32\lib\site-packages\telebot\__init__.py", line 513, in __threaded_polling
raise e
File "C:\Users\pc\AppData\Local\Programs\Python\Python38-32\lib\site-packages\telebot\__init__.py", line 478, in __threaded_polling
polling_thread.raise_exceptions()
File "C:\Users\pc\AppData\Local\Programs\Python\Python38-32\lib\site-packages\telebot\util.py", line 88, in raise_exceptions
raise self.exception_info
File "C:\Users\pc\AppData\Local\Programs\Python\Python38-32\lib\site-packages\telebot\util.py", line 69, in run
task(*args, **kwargs)
File "C:\Users\pc\AppData\Local\Programs\Python\Python38-32\lib\site-packages\telebot\__init__.py", line 295, in __retrieve_updates
updates = self.get_updates(offset=(self.last_update_id + 1), timeout=timeout, long_polling_timeout = long_polling_timeout)
File "C:\Users\pc\AppData\Local\Programs\Python\Python38-32\lib\site-packages\telebot\__init__.py", line 265, in get_updates
json_updates = apihelper.get_updates(self.token, offset, limit, timeout, allowed_updates, long_polling_timeout)
File "C:\Users\pc\AppData\Local\Programs\Python\Python38-32\lib\site-packages\telebot\apihelper.py", line 248, in get_updates
return _make_request(token, method_url, params=payload)
File "C:\Users\pc\AppData\Local\Programs\Python\Python38-32\lib\site-packages\telebot\apihelper.py", line 107, in _make_request
result = _get_req_session().request(
File "C:\Users\pc\AppData\Local\Programs\Python\Python38-32\lib\site-packages\requests\sessions.py", line 542, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\pc\AppData\Local\Programs\Python\Python38-32\lib\site-packages\requests\sessions.py", line 655, in send
r = adapter.send(request, **kwargs)
File "C:\Users\pc\AppData\Local\Programs\Python\Python38-32\lib\site-packages\requests\adapters.py", line 514, in send
raise SSLError(e, request=request)
requests.exceptions.SSLError: HTTPSConnectionPool(host='api.telegram.org', port=443): Max retries exceeded with url: /bot1494641875:AAEPX72rP7YFT0ZZzASh0cn9rAw6l8PUMk0/getUpdates?offset=1&timeout=20 (Caused by SSLError(SSLError(1, '[SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:1123)')))
Connection is OK. Token is OK. It works manually when I browse https://api.telegram.org/botToken/getme and it works fine.
Other site for example google.com are OK too and works fine.
I would be glad to know what is the problem.
Actually because error I receive is too much a lot, stackoverflow does not allow me to post the question and wants me to add some more details, so I wrote this paragraph which is not related to the question and is just some words.
Do you have connected your telegram with a proxy? I can't solve your problem but you can follow this post: Problem connecting telegram bot through Python

python ssl error from gs cloud client MaxRetryError or WRONG_VERSION_NUMBER

I have a code that runs 24/7 (while true on the main.py file) with multiprocessing (12 workers) that accesses gcloud on a regular basis (listing buckets, downloading etc.)
I dont think its a regular ssl error because I get ssl error only one time throughout a day long run and every subprocess access their own gs client.
I dont really understand why the error happens or how to fix it since the same traceback gives me two types of errors.
Also worked everything until i increased the amount of my workers.
The way i create my client:
def get_gs_bucket():
storage_client = storage.Client.from_service_account_json(JSON_CREDENTIALS_PATH)
gs_bucket_name = BUCKET_NAME
return storage_client.get_bucket(gs_bucket_name)
the function that's mentioned in the error:
def list_gs_dir(prefix=''):
sleep_random_prevent_gs_over_api_usage()
return gs_bucket.list_blobs(delimiter='/', prefix=prefix)._get_next_page_response()['prefixes']
my error:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 601, in urlopen
chunked=chunked)
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 387, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 383, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.6/http/client.py", line 1373, in getresponse
response.begin()
File "/usr/lib/python3.6/http/client.py", line 311, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.6/http/client.py", line 272, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/lib/python3.6/socket.py", line 586, in readinto
return self._sock.recv_into(b)
File "/usr/lib/python3.6/ssl.py", line 1012, in recv_into
return self.read(nbytes, buffer)
File "/usr/lib/python3.6/ssl.py", line 874, in read
return self._sslobj.read(len, buffer)
File "/usr/lib/python3.6/ssl.py", line 631, in read
v = self._sslobj.read(len, buffer)
ssl.SSLError: [SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:2309)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 440, in send
timeout=timeout
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 639, in urlopen
_stacktrace=sys.exc_info()[2])
File "/usr/lib/python3/dist-packages/urllib3/util/retry.py", line 398, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='www.googleapis.com', port=443): Max retries exceeded with url: <url> (Caused by SSLError(SSLError(1, '[SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:2309)'),))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/etl/work_wish/etl/etl/utils.py", line 97, in inner
return func(*args, **kwargs)
File "/opt/etl/work_wish/etl/etl/input_bucket.py", line 119, in get_recent_sstables_group_file_per_node
sstables_ts_pairs = get_sstables_ts_pairs(node, table_type, last_ts)
File "/opt/etl/work_wish/etl/etl/input_bucket.py", line 129, in get_sstables_ts_pairs
gs_path_prefixes = list_gs_dir(prefix=node)
File "/opt/etl/work_wish/etl/etl/input_bucket.py", line 35, in list_gs_dir
return gs_bucket.list_blobs(delimiter='/', prefix=prefix)._get_next_page_response()['prefixes']
File "/usr/local/lib/python3.6/dist-packages/google/api_core/page_iterator.py", line 419, in _get_next_page_response
method=self._HTTP_METHOD, path=self.path, query_params=params
File "/usr/local/lib/python3.6/dist-packages/google/cloud/_http.py", line 315, in api_request
target_object=_target_object,
File "/usr/local/lib/python3.6/dist-packages/google/cloud/_http.py", line 192, in _make_request
return self._do_request(method, url, headers, data, target_object)
File "/usr/local/lib/python3.6/dist-packages/google/cloud/_http.py", line 221, in _do_request
return self.http.request(url=url, method=method, headers=headers, data=data)
File "/usr/local/lib/python3.6/dist-packages/google/auth/transport/requests.py", line 470, in request
**kwargs
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 520, in request
resp = self.send(prep, **send_kwargs)
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 630, in send
r = adapter.send(request, **kwargs)
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 506, in send
raise SSLError(e, request=request)
requests.exceptions.SSLError: HTTPSConnectionPool(host='www.googleapis.com', port=443): Max retries exceeded with url: <url> (Caused by SSLError(SSLError(1, '[SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:2309)'),))

Python urllib3 - Crawling webpages failed at a specific webpage only from google cloud vm

I have built a crawler script that download webpages using urllib3 and python3. I ran the crawler in a Google Cloud Virtual Machine instance and tried it with many different urls. I found a url that i couldn't download on the VM instance but succeeded to download on my personal Linux.
The code that i've tried on my computer and worked:
>>> import urllib3
>>> pool = urllib3.PoolManager()
>>> response = pool.request('GET', 'http://www.azlyrics.com/lyrics/drake/dowhatyoudo.html')
>>> response.status
200
I have tried the exact same code on the VM instance using ssh and this exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py", line 594, in urlopen
chunked=chunked)
File "/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py", line 391, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py", line 387, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.5/http/client.py", line 1197, in getresponse
response.begin()
File "/usr/lib/python3.5/http/client.py", line 297, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.5/http/client.py", line 266, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/local/lib/python3.5/dist-packages/urllib3/request.py", line 66, in request
**urlopen_kw)
File "/usr/local/lib/python3.5/dist-packages/urllib3/request.py", line 87, in request_encode_url
return self.urlopen(method, url, **extra_kw)
File "/usr/local/lib/python3.5/dist-packages/urllib3/poolmanager.py", line 244, in urlopen
response = conn.urlopen(method, u.request_uri, **kw)
File "/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py", line 671, in urlopen
release_conn=release_conn, **response_kw)
File "/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py", line 671, in urlopen
release_conn=release_conn, **response_kw)
File "/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py", line 671, in urlopen
release_conn=release_conn, **response_kw)
File "/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py", line 643, in urlopen
_stacktrace=sys.exc_info()[2])
File "/usr/local/lib/python3.5/dist-packages/urllib3/util/retry.py", line 303, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='www.azlyrics.com', port=80): Max retries exceeded with url: /lyrics/drake/dowhatyoudo.html (Caused by ProtocolError('Connection aborted.', RemoteDisconnected('Remote
end closed connection without response',)))
I have checked the VM dashboard and I didn't find any firewall or other restrictions that I may have mistakenly defined.
What could it possibly be?

Categories