I already checked if proxy settings were enabled, reset winsock, uninstall firewall and antivirus, and I still get the same error message, but if I type the url in Firefox or IE it works just fine. Thanks for any help
Here is the script:
import urllib2
import ssl
gcontext = ssl._create_unverified_context()
url = 'https://192.168.1.1:8000/cgi-bin/login.html'
html = urllib2.urlopen(url=url,context=gcontext).read()
print html
and here is the traceback:
File "get_url.py", line 6, in <module>
f = urllib2.urlopen("https://192.168.1.1:8000/cgi-bin/login.html",context=gcontext, timeout=3000)
File "/usr/lib/python2.7/urllib2.py", line 154, in urlopen
return opener.open(url, data, timeout)
File "/usr/lib/python2.7/urllib2.py", line 435, in open
response = meth(req, response)
File "/usr/lib/python2.7/urllib2.py", line 548, in http_response
'http', request, response, code, msg, hdrs)
File "/usr/lib/python2.7/urllib2.py", line 467, in error
result = self._call_chain(*args)
File "/usr/lib/python2.7/urllib2.py", line 407, in _call_chain
result = func(*args)
File "/usr/lib/python2.7/urllib2.py", line 654, in http_error_302
return self.parent.open(new, timeout=req.timeout)
File "/usr/lib/python2.7/urllib2.py", line 429, in open
response = self._open(req, data)
File "/usr/lib/python2.7/urllib2.py", line 447, in _open
'_open', req)
File "/usr/lib/python2.7/urllib2.py", line 407, in _call_chain
result = func(*args)
File "/usr/lib/python2.7/urllib2.py", line 1241, in https_open
context=self._context)
File "/usr/lib/python2.7/urllib2.py", line 1198, in do_open
raise URLError(err)
urllib2.URLError: <urlopen error [Errno 111] Connection refused>
Related
I am using with python urllib2 to a connect HTTP server. Sometimes I get the response: httplib.BadStatusLine: ''.
My code :
response = None
try:
request = urllib2.Request(http_url,params)
response = urllib2.urlopen(request,timeout=5000)
return str(response.read())
except urllib2.HTTPError :
return ""
except urllib2.URLError:
return ""
response error :
File "/usr/lib64/python2.7/urllib2.py", line 154, in urlopen
return opener.open(url, data, timeout)
File "/usr/lib64/python2.7/urllib2.py", line 431, in open
response = self._open(req, data)
File "/usr/lib64/python2.7/urllib2.py", line 449, in _open
'_open', req)
File "/usr/lib64/python2.7/urllib2.py", line 409, in _call_chain
result = func(*args)
File "/usr/lib64/python2.7/urllib2.py", line 1244, in http_open
return self.do_open(httplib.HTTPConnection, req)
File "/usr/lib64/python2.7/urllib2.py", line 1217, in do_open
r = h.getresponse(buffering=True)
File "/usr/lib64/python2.7/httplib.py", line 1051, in getresponse
response.begin()
File "/usr/lib64/python2.7/httplib.py", line 415, in begin
version, status, reason = self._read_status()
File "/usr/lib64/python2.7/httplib.py", line 379, in _read_status
raise BadStatusLine(line)
httplib.BadStatusLine: ''
I don't to know how to fix this bug? And I don't to know why I get an error response.
When I run this script I get a ton of errors.
import urllib, urllib2
proxy = urllib2.ProxyHandler({
'http': '127.0.0.1',
'https': '127.0.0.1'
})
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
# this way both http and https requests go through the proxy
urllib2.urlopen('http://www.google.com')
urllib2.urlopen('https://www.google.com')
I don't really understand what these errors are, hence why I am asking. Here they are:
Traceback (most recent call last):
File "C:\Python27\Craig.py", line 10, in <module>
urllib2.urlopen('http://www.google.com')
File "C:\Python27\lib\urllib2.py", line 127, in urlopen
return _opener.open(url, data, timeout)
File "C:\Python27\lib\urllib2.py", line 410, in open
response = meth(req, response)
File "C:\Python27\lib\urllib2.py", line 523, in http_response
'http', request, response, code, msg, hdrs)
File "C:\Python27\lib\urllib2.py", line 448, in error
return self._call_chain(*args)
File "C:\Python27\lib\urllib2.py", line 382, in _call_chain
result = func(*args)
File "C:\Python27\lib\urllib2.py", line 531, in http_error_default
raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
urllib2.HTTPError: HTTP Error 501: Not Implemented
Update:
After I added the ports I got these errors:
Traceback (most recent call last):
File "C:\Python27\Craig.py", line 10, in <module>
urllib2.urlopen('http://www.google.com')
File "C:\Python27\lib\urllib2.py", line 127, in urlopen
return _opener.open(url, data, timeout)
File "C:\Python27\lib\urllib2.py", line 404, in open
response = self._open(req, data)
File "C:\Python27\lib\urllib2.py", line 422, in _open
'_open', req)
File "C:\Python27\lib\urllib2.py", line 382, in _call_chain
result = func(*args)
File "C:\Python27\lib\urllib2.py", line 1214, in http_open
return self.do_open(httplib.HTTPConnection, req)
File "C:\Python27\lib\urllib2.py", line 1184, in do_open
raise URLError(err)
urllib2.URLError: <urlopen error [Errno 10061] No connection could be made because the target machine actively refused it>
Are you running a local proxy?
If you are, it's probably not running on port 80. The default port is 3128.
proxy = urllib2.ProxyHandler({
'http': 'http://127.0.0.1:3128/',
'https': 'http://127.0.0.1:3128/'
})
I'm running a scraper that's going through a few domains and it's getting hung up on http://www.1000markets.com/
Checking from different sources it seems to be down. That's totally fine, but I'm getting the error mentioned in the title.
How can I escape this? I'm using HTTPerror and URLerror but it's still getting hung up.
Any help on this would be great
def get_html(link):
import urllib2
from urllib2 import Request, urlopen, URLError, HTTPError
try:
res = urllib2.urlopen(link)
html = res.read()
except URLError as e:
return link
except HTTPError as e:
return link
edit: Attached is the error
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 127, in urlopen
return _opener.open(url, data, timeout)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 404, in open
response = self._open(req, data)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 422, in _open
'_open', req)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 382, in _call_chain
result = func(*args)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 1214, in http_open
return self.do_open(httplib.HTTPConnection, req)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 1187, in do_open
r = h.getresponse(buffering=True)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py", line 1045, in getresponse
response.begin()
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py", line 409, in begin
version, status, reason = self._read_status()
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py", line 365, in _read_status
line = self.fp.readline(_MAXLINE + 1)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/socket.py", line 476, in readline
data = self._sock.recv(self._rbufsize)
socket.error: [Errno 54] Connection reset by peer
I try to send post request with json body with urllib2:
request = urllib2.Request('http://localhost:8090/api/', jdata, {'Content-Type': 'application/json'})
f = urllib2.urlopen(request)
But get exception:
File "/usr/lib/python2.7/urllib2.py", line 127, in urlopen
return _opener.open(url, data, timeout)
File "/usr/lib/python2.7/urllib2.py", line 407, in open
response = meth(req, response)
File "/usr/lib/python2.7/urllib2.py", line 520, in http_response
'http', request, response, code, msg, hdrs)
File "/usr/lib/python2.7/urllib2.py", line 439, in error
result = self._call_chain(*args)
File "/usr/lib/python2.7/urllib2.py", line 379, in _call_chain
result = func(*args)
File "/usr/lib/python2.7/urllib2.py", line 626, in http_error_302
return self.parent.open(new, timeout=req.timeout)
File "/usr/lib/python2.7/urllib2.py", line 401, in open
response = self._open(req, data)
File "/usr/lib/python2.7/urllib2.py", line 419, in _open
'_open', req)
File "/usr/lib/python2.7/urllib2.py", line 379, in _call_chain
result = func(*args)
File "/usr/lib/python2.7/urllib2.py", line 1211, in http_open
return self.do_open(httplib.HTTPConnection, req)
File "/usr/lib/python2.7/urllib2.py", line 1150, in do_open
h = http_class(host, timeout=req.timeout) # will parse host:port
File "/usr/lib/python2.7/httplib.py", line 693, in __init__
self._set_hostport(host, port)
File "/usr/lib/python2.7/httplib.py", line 721, in _set_hostport
raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
httplib.InvalidURL: nonnumeric port: '8090��>��hbWb�ba'
How can i fix it?
Thank you.
It seems that the urllib2 functions doesn't parse right your request. Why don't you try curl?
import pycurl
c = pycurl.Curl()
c.setopt(c.URL, 'http://myfavpizzaplace.com/order')
c.setopt(c.POSTFIELDS, 'pizza=Quattro+Stagioni&extra=cheese')
c.setopt(c.VERBOSE, True)
c.perform()
example from http://www.angryobjects.com/2011/10/15/http-with-python-pycurl-by-example/
For some reason I can get this to work, using single proxy everything seems fine.
#This works
import socks
import urllib2
socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, '68.xx.193.xx', 666)
socks.wrapmodule(urllib2)
print urllib2.urlopen('http://automation.whatismyip.com/n09230945.asp').read()
&
#This doesn't
import socks
import urllib2
proxies=['68.xx.193.xx','xx.178.xx.70','98.xx.84.xx','83.xx.86.xx']
ports=[666,1080,859,910]
for i in range(len(proxies)):
socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, repr(proxies[i]), ports[i])
socks.wrapmodule(urllib2)
print urllib2.urlopen('http://automation.whatismyip.com/n09230945.asp').read()
Error:
Traceback (most recent call last):
File "/home/zer0/Aptana Studio 3 Workspace/sy/src/test.py", line 38, in <module>
print urllib2.urlopen('http://automation.whatismyip.com/n09230945.asp').read()
File "/usr/lib/python2.7/urllib2.py", line 126, in urlopen
return _opener.open(url, data, timeout)
File "/usr/lib/python2.7/urllib2.py", line 391, in open
response = self._open(req, data)
File "/usr/lib/python2.7/urllib2.py", line 409, in _open
'_open', req)
File "/usr/lib/python2.7/urllib2.py", line 369, in _call_chain
result = func(*args)
File "/usr/lib/python2.7/urllib2.py", line 1185, in http_open
return self.do_open(httplib.HTTPConnection, req)
File "/usr/lib/python2.7/urllib2.py", line 1160, in do_open
raise URLError(err)
urllib2.URLError: <urlopen error [Errno -5] No address associated with hostname>
Run this:
proxies=['68.xx.193.xx','xx.178.xx.70','98.xx.84.xx','83.xx.86.xx']
ports=[666,1080,859,910]
for i in range(len(proxies)):
print (repr(proxies[i]), ports[i])
You'll get
("'68.xx.193.xx'", 666)
("'xx.178.xx.70'", 1080)
("'98.xx.84.xx'", 859)
("'83.xx.86.xx'", 910)
You're adding quotes you don't want with the repr call, so urllib2 thinks it's a hostname instead of an IP address. Get rid of it and you should be fine.