So im trying to use python to send a post to a url, i have basically copied everything from my network console in firefox yet i can't get it to work.
import requests
import json
url = 'https://www.example.com/example?handle'
cookie = {'UID':'b56f14d8-02b1-4ae8-ad5e-3485cf0abdbe'}
data ={'example':'example'}
headers = {'Accept':'*/*',
'Accept-Encoding':'gzip, deflate, br',
'Accept-Language':'en-US,en;q=0.5',
'Connection':'keep-alive',
'Content-Length':'78',
'Content-Type':'text/plain;charset=UTF-8',
'Cookie':'UID=b56f14d8-02b1-4ae8-ad5e-3485cf0abdbe',
'Host':'www.example.com',
'Referer':'https:/example.com/example',
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0'}
req = requests.post(url, cookies=cookie, params=json.dumps(data), headers=headers)
print(req)
I have replaced the actual data and url with example.
After running this, i get a big long error:
Traceback (most recent call last):
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32 \lib\site-packages\urllib3\connectionpool.py", line 600, in urlopen
chunked=chunked)
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\site-packages\urllib3\connectionpool.py", line 384, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\site-packages\urllib3\connectionpool.py", line 380, in _make_request
httplib_response = conn.getresponse()
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\http\client.py", line 1321, in getresponse
response.begin()
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\http\client.py", line 296, in begin
version, status, reason = self._read_status()
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\http\client.py", line 257, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\socket.py", line 589, in readinto
return self._sock.recv_into(b)
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\ssl.py", line 1052, in recv_into
return self.read(nbytes, buffer)
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\ssl.py", line 911, in read
return self._sslobj.read(len, buffer)
ConnectionResetError: [WinError 10054] An existing connection was forcibly closed by the remote host
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\site-packages\requests\adapters.py", line 449, in send
timeout=timeout
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\site-packages\urllib3\connectionpool.py", line 638, in urlopen
_stacktrace=sys.exc_info()[2])
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\site-packages\urllib3\util\retry.py", line 367, in increment
raise six.reraise(type(error), error, _stacktrace)
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\site-packages\urllib3\packages\six.py", line 685, in reraise
raise value.with_traceback(tb)
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\site-packages\urllib3\connectionpool.py", line 600, in urlopen
chunked=chunked)
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\site-packages\urllib3\connectionpool.py", line 384, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\site-packages\urllib3\connectionpool.py", line 380, in _make_request
httplib_response = conn.getresponse()
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\http\client.py", line 1321, in getresponse
response.begin()
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\http\client.py", line 296, in begin
version, status, reason = self._read_status()
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\http\client.py", line 257, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\socket.py", line 589, in readinto
return self._sock.recv_into(b)
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\ssl.py", line 1052, in recv_into
return self.read(nbytes, buffer)
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\ssl.py", line 911, in read
return self._sslobj.read(len, buffer)
urllib3.exceptions.ProtocolError: ('Connection aborted.', ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:/Users/Kaiser/Documents/python ddoser.py", line 19, in <module>
req = requests.post(url, cookies=cookie, params=json.dumps(data).encode('UTF-8'), headers=headers)
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\site-packages\requests\api.py", line 116, in post
return request('post', url, data=data, json=json, **kwargs)
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\site-packages\requests\api.py", line 60, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\site-packages\requests\sessions.py", line 533, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\site-packages\requests\sessions.py", line 646, in send
r = adapter.send(request, **kwargs)
File "C:\Users\Kaiser\AppData\Local\Programs\Python\Python37-32\lib\site-packages\requests\adapters.py", line 498, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None))
Does anyone know about what i could be doing wrong. Sorry if its a rookie mistake.
This error occurs when the web server rejects your connection. A web server can reject your connection for various reasons. Most likely, you did not properly mimic the HTTP request the web server was expecting.
You do not have to manually fill in all of the HTTP parameters such content length. The library automatically fills in these fields for you. If you plan to make subsequent HTTP requests using the same cookies, consider creating a session object, which stores and handles the cookies for you:
session = requests.Session()
session.post(url, data=payload)
Also, realize the difference between the data parameter and the params parameter. Since you are sending a POST request, chances are that the web server is expecting the data to be passed via the data parameter (which places the data in the request body rather than the query string in the URL). Another reason a web server may reject your connection is because you sent a HTTP request rather than a HTTPS request, so always ensure everything is correct on your end.
Below the code that i use to send post information and read the response,
import urllib2
from urllib2 import URLError, HTTPError
import urllib
import ssl
mydata=[('funcion',funcion),('codigo',codigo)] #The first is the var name the second is the value
headers = {"Content-type", "application/x-www-form-urlencoded","User-Agent", "Mozilla/5.0",}
handler=urllib2.HTTPHandler(debuglevel=1)
opener = urllib2.build_opener(handler)
urllib2.install_opener(opener)
mydata=urllib.urlencode(mydata)
path=url #the url you want to POST to
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE #In my case i need accept certificates
req=urllib2.Request(path, mydata)
req.add_header("Content-type", "application/x-www-form-urlencoded")
req.add_header("User-Agent", "Mozilla/5.0")
page=urllib2.urlopen(req).read()
Related
I'm trying to GET data from an external service using Python from a Docker container.
I need to reach two different endpoints, let's say:
endpointA = "https://myhost:1234/resA" # Requests take about 1 minute to complete
endpointB = "https://myhost:1234/resB" # Requests take about 8 minutes to complete
I tried to use two different libraries, Requests:
import requests
headers = {'Authorization': 'Basic XXXXXXXXXXXXXXXXXXXXXX', 'Content-Type': 'application/json'}
response = requests.get(endpointA, verify=False, headers=headers)
and Pycurl:
import pycurl
headers = ['Authorization: Basic XXXXXXXXXXXXXXXXXXXXXX', 'Content-Type: application/json']
buffer = BytesIO()
c = pycurl.Curl()
c.setopt(c.VERBOSE, True)
c.setopt(c.URL, endpointA)
c.setopt(c.WRITEDATA, buffer)
c.setopt(c.SSL_VERIFYPEER, False)
c.setopt(c.SSL_VERIFYHOST, False)
c.setopt(c.HTTPHEADER, headers)
c.perform()
c.close()
My code runs without problems from local (both endpointA and endpointB return).
When run inside a Docker container, requests to endpointA work but request to endpointB raise exceptions after 300 seconds:
in the case of requests:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 670, in urlopen
httplib_response = self._make_request(
File "/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 426, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 421, in _make_request
httplib_response = conn.getresponse()
File "/usr/local/lib/python3.8/http/client.py", line 1347, in getresponse
response.begin()
File "/usr/local/lib/python3.8/http/client.py", line 307, in begin
version, status, reason = self._read_status()
File "/usr/local/lib/python3.8/http/client.py", line 276, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/requests/adapters.py", line 439, in send
resp = conn.urlopen(
File "/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 726, in urlopen
retries = retries.increment(
File "/usr/local/lib/python3.8/site-packages/urllib3/util/retry.py", line 403, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/usr/local/lib/python3.8/site-packages/urllib3/packages/six.py", line 734, in reraise
raise value.with_traceback(tb)
File "/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 670, in urlopen
httplib_response = self._make_request(
File "/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 426, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 421, in _make_request
httplib_response = conn.getresponse()
File "/usr/local/lib/python3.8/http/client.py", line 1347, in getresponse
response.begin()
File "/usr/local/lib/python3.8/http/client.py", line 307, in begin
version, status, reason = self._read_status()
File "/usr/local/lib/python3.8/http/client.py", line 276, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
in the case of pycurl:
Traceback (most recent call last):
File "test.py", line 230, in get_data
c.perform()
pycurl.error: (52, 'Empty reply from server')
This only happens from Docker, as from my console everything works fine.
I tried both python:3.6 and python:3.8 Docker images.
I also tried to cURL both endpoints from inside the container and the server replies correctly.
You can verify whether docker is terminating the connection by using nc -l outside the container and nc in the container to connect to the first nc, then leaving them both alone for 10 minutes and then sending something to verify the connection is still up.
To troubleshoot pycurl, see http://pycurl.io/docs/latest/troubleshooting.html#transfer-related-issues.
I am trying to launch my script to PA and while it works locally it does not work on PA and I cannot figure out why. I have the paid version of PA so I can access this site that is not whitelisted. Here is my script:
with requests.Session() as session:
# initates the login session
if os.path.exists(document_report + '.pdf'):
# initates the login session
post = session.post(POST_LOGIN_URL, data=payload, headers = dict(referrer = 'https://www.example.com/loginpage'))
#This URL is the page you actually want to pull down with requests.
REQUEST_URL = f'https://www.example.com/{stuff}/{document}'
#gets the request URL
r = session.get(REQUEST_URL)
# saves to directory
with open('./digests/1.pdf', 'wb') as j:
j.write(r.content)
j.close()
else:
post = session.post(POST_LOGIN_URL, data=payload, headers = dict(referrer = 'https://www.example.com/loginpage'))
#This URL is the page you actually want to pull down with requests.
REQUEST_URL = f'https://www.example.com/{stuff}/{document}'
#gets the request URL
r = session.get(REQUEST_URL)
# saves to directory
with open(document_report + '.pdf', 'wb') as j:
j.write(r.content)
j.close()
When I run the code, I get the following message:
Traceback (most recent call last):
File "/usr/lib/python3.7/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/usr/lib/python3.7/site-packages/urllib3/connectionpool.py", line 384, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "/usr/lib/python3.7/site-packages/urllib3/connectionpool.py", line 380, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.7/http/client.py", line 1344, in getresponse
response.begin()
File "/usr/lib/python3.7/http/client.py", line 306, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.7/http/client.py", line 275, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3.7/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/usr/lib/python3.7/site-packages/urllib3/connectionpool.py", line 638, in urlopen
_stacktrace=sys.exc_info()[2])
File "/usr/lib/python3.7/site-packages/urllib3/util/retry.py", line 368, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/usr/lib/python3.7/site-packages/urllib3/packages/six.py", line 685, in reraise
raise value.with_traceback(tb)
File "/usr/lib/python3.7/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/usr/lib/python3.7/site-packages/urllib3/connectionpool.py", line 384, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "/usr/lib/python3.7/site-packages/urllib3/connectionpool.py", line 380, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.7/http/client.py", line 1344, in getresponse
response.begin()
File "/usr/lib/python3.7/http/client.py", line 306, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.7/http/client.py", line 275, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/legalaide/emailProcessorV2.py", line 567, in <module>
post = session.post(POST_LOGIN_URL, data=payload, headers = dict(referrer = 'https://www.example.com/loginpage'))
File "/usr/lib/python3.7/site-packages/requests/sessions.py", line 581, in post
return self.request('POST', url, data=data, json=json, **kwargs)
File "/usr/lib/python3.7/site-packages/requests/sessions.py", line 533, in request
resp = self.send(prep, **send_kwargs)
File "/usr/lib/python3.7/site-packages/requests/sessions.py", line 646, in send
r = adapter.send(request, **kwargs)
File "/usr/lib/python3.7/site-packages/requests/adapters.py", line 498, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
I tried it again locally, and it still works. Any insight into a solution would be amazing!
I made simple HTTP Server using BaseHTTPRequestHandler. The problem is, that when I want to post some data using requests from client, I get ConnectionError. I did simple request from requests lib documentation. Also interesting thing is, that HTTP Server will receive data from client and print it to console. I don't understand how its possible.
Client:
def post_data():
"""Client method"""
json_data = {
'sender': 'User',
'receiver': 'MY_SERVER',
'message': 'Hello server! Sending some data.'}
data_headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
data_payload = json.dumps(json_data)
try:
post = requests.post('http://localhost:8080/post', data=data_payload,
headers=data_headers)
print(post.status_code)
except ConnectionError as e:
print("CONNECTION ERROR: ")
print(e)
HTTP Server:
def do_POST(self):
"""Server method"""
self.send_response(200)
print("Receiving new data ...")
content_length = int(self.headers['Content-Length'])
post_data = self.rfile.read(content_length)
print(post_data)
Server result:
C:\Users\mypc\Projects\PythonFiles\httpserver>python server.py
Fri Jan 5 01:09:12 2018: HTTP Server started on port 8080.
127.0.0.1 - - [05/Jan/2018 01:09:21] "POST /post HTTP/1.1" 200 -
Receiving new data ...
b'{"sender": "User", "receiver": "MY_SERVER", "message": "Hello server! Sending some data."}'
Client result:
C:\Users\mypc\Projects\PythonFiles\httpserver>python client.py
CONNECTION ERROR:
('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))
C:\Users\mypc\Projects\PythonFiles\httpserver>
Error without exception block:
Traceback (most recent call last):
File "C:\Python36\lib\site-packages\urllib3\connectionpool.py", line 601, in urlopen
chunked=chunked)
File "C:\Python36\lib\site-packages\urllib3\connectionpool.py", line 387, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "C:\Python36\lib\site-packages\urllib3\connectionpool.py", line 383, in _make_request
httplib_response = conn.getresponse()
File "C:\Python36\lib\http\client.py", line 1331, in getresponse
response.begin()
File "C:\Python36\lib\http\client.py", line 297, in begin
version, status, reason = self._read_status()
File "C:\Python36\lib\http\client.py", line 266, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Python36\lib\site-packages\requests\adapters.py", line 440, in send
timeout=timeout
File "C:\Python36\lib\site-packages\urllib3\connectionpool.py", line 639, in urlopen
_stacktrace=sys.exc_info()[2])
File "C:\Python36\lib\site-packages\urllib3\util\retry.py", line 357, in increment
raise six.reraise(type(error), error, _stacktrace)
File "C:\Python36\lib\site-packages\urllib3\packages\six.py", line 685, in reraise
raise value.with_traceback(tb)
File "C:\Python36\lib\site-packages\urllib3\connectionpool.py", line 601, in urlopen
chunked=chunked)
File "C:\Python36\lib\site-packages\urllib3\connectionpool.py", line 387, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "C:\Python36\lib\site-packages\urllib3\connectionpool.py", line 383, in _make_request
httplib_response = conn.getresponse()
File "C:\Python36\lib\http\client.py", line 1331, in getresponse
response.begin()
File "C:\Python36\lib\http\client.py", line 297, in begin
version, status, reason = self._read_status()
File "C:\Python36\lib\http\client.py", line 266, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "client.py", line 137, in <module>
start_parser()
File "client.py", line 101, in start_parser
send_requests(args.get, args.post)
File "client.py", line 51, in send_requests
post_data()
File "client.py", line 129, in post_data
headers=data_headers)
File "C:\Python36\lib\site-packages\requests\api.py", line 112, in post
return request('post', url, data=data, json=json, **kwargs)
File "C:\Python36\lib\site-packages\requests\api.py", line 58, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Python36\lib\site-packages\requests\sessions.py", line 508, in request
resp = self.send(prep, **send_kwargs)
File "C:\Python36\lib\site-packages\requests\sessions.py", line 618, in send
r = adapter.send(request, **kwargs)
File "C:\Python36\lib\site-packages\requests\adapters.py", line 490, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))
It looks like the server is terminating the connection early without sending a full response. I've skimmed the docs and I think this is the problem (emphasis added):
send_response(code, message=None)
Adds a response header to the
headers buffer and logs the accepted request. The HTTP response line
is written to the internal buffer, followed by Server and Date
headers. The values for these two headers are picked up from the
version_string() and date_time_string() methods, respectively. If the
server does not intend to send any other headers using the
send_header() method, then send_response() should be followed by an
end_headers() call.
Changed in version 3.3: Headers are stored to an internal buffer and
end_headers() needs to be called explicitly.
So you probably just need to add the call to end_headers. If you were reading an old example (prior to Python 3.3) this wouldn't have been needed.
You can just put end_headers as Peter Gibson said
self.send_header('Content-Type', 'blabla' )
self.end_headers()
Language Ver: Python 3.6.3
IDE Ver: PyCharm 2017.2.3
I was trying to parse a weather website to print weather for a place. As I am learning Python, previously I used urllib.request.urlopen(url).read() and it worked. Now, I am modifying the code to BeautifulSoup4 and requests module. Below is my code:
from bs4 import *
import requests
url = "https://www.accuweather.com/en/in/dhenkanal/189844/weather-forecast/189844"
data = requests.get(url)
soup = BeautifulSoup(data.text, "html.parser")
print(soup.find('div', {'class': 'info'}))
But each time I try to run the code it gives me following error:
Traceback (most recent call last):
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\site-packages\urllib3\connectionpool.py", line 601, in urlopen
chunked=chunked)
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\site-packages\urllib3\connectionpool.py", line 387, in _make_request
six.raise_from(e, None)
File "", line 2, in raise_from
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\site-packages\urllib3\connectionpool.py", line 383, in _make_request
httplib_response = conn.getresponse()
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 1331, in getresponse
response.begin()
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 297, in begin
version, status, reason = self._read_status()
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 258, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\socket.py", line 586, in readinto
return self._sock.recv_into(b)
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\ssl.py", line 1009, in recv_into
return self.read(nbytes, buffer)
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\ssl.py", line 871, in read
return self._sslobj.read(len, buffer)
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\ssl.py", line 631, in read
v = self._sslobj.read(len, buffer)
TimeoutError: [WinError 10060] A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\site-packages\requests\adapters.py", line 440, in send
timeout=timeout
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\site-packages\urllib3\connectionpool.py", line 639, in urlopen
_stacktrace=sys.exc_info()[2])
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\site-packages\urllib3\util\retry.py", line 357, in increment
raise six.reraise(type(error), error, _stacktrace)
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\site-packages\urllib3\packages\six.py", line 685, in reraise
raise value.with_traceback(tb)
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\site-packages\urllib3\connectionpool.py", line 601, in urlopen
chunked=chunked)
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\site-packages\urllib3\connectionpool.py", line 387, in _make_request
six.raise_from(e, None)
File "", line 2, in raise_from
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\site-packages\urllib3\connectionpool.py", line 383, in _make_request
httplib_response = conn.getresponse()
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 1331, in getresponse
response.begin()
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 297, in begin
version, status, reason = self._read_status()
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 258, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\socket.py", line 586, in readinto
return self._sock.recv_into(b)
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\ssl.py", line 1009, in recv_into
return self.read(nbytes, buffer)
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\ssl.py", line 871, in read
return self._sslobj.read(len, buffer)
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\ssl.py", line 631, in read
v = self._sslobj.read(len, buffer)
urllib3.exceptions.ProtocolError: ('Connection aborted.', TimeoutError(10060, 'A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond', None, 10060, None))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "E:/Projects/Python/Practice/Practice1.py", line 5, in
data = requests.get(url)
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\site-packages\requests\api.py", line 72, in get
return request('get', url, params=params, **kwargs)
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\site-packages\requests\api.py", line 58, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\site-packages\requests\sessions.py", line 508, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\site-packages\requests\sessions.py", line 618, in send
r = adapter.send(request, **kwargs)
File "C:\Users\Nrusingh\AppData\Local\Programs\Python\Python36-32\lib\site-packages\requests\adapters.py", line 490, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', TimeoutError(10060, 'A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond', None, 10060, None))
Process finished with exit code 1
What is this error and how to correct it? And why it worked in urllib, but not in requests?
I used your code straight up and I got the same error then I followed how the requests are sent in browser. Some servers don't respond if expected headers are not sent with request that they use as part of backend processing. Turns out the server was looking for a header called user-agent usually used to determine what client the request is from. Now, amended code below which works!
from bs4 import *
import requests
url = "https://www.accuweather.com/en/in/dhenkanal/189844/weather-forecast/189844"
headers = {'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36'}
data = requests.get(url, headers=headers)
soup = BeautifulSoup(data.text, "html.parser")
Now you can play with your soup!
You can in fact pass more headers like accept, dnt, pragma, accept-language, cache-control etc. Explanation of these http headers are for another question, another time. Hope it helps :)
Try increasing the timeout parameter of your requests.get method :
requests.get(url, headers=headers, timeout=5)
But if your script is being blocked by the server to prevent scrapping attempts . If this is the case you can try faking a web browser by setting appropriate headers .
{"User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2.8) Gecko/20100722 Firefox/3.6.8 GTB7.1 (.NET CLR 3.5.30729)", "Referer": "http://example.com"}
your final code
import requests
url = "https://www.accuweather.com/en/in/dhenkanal/189844/weather-forecast/189844"
headers = {"User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2.8) Gecko/20100722 Firefox/3.6.8 GTB7.1 (.NET CLR 3.5.30729)", "Referer": "http://example.com"}
data = requests.get(url,headers=headers,timeout=5)
import json
import string
import socket
import requests
from bs4 import BeautifulSoup
# Default header to be used first.
headers={"User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36"}
# Create a session using requests to log in.
with requests.Session() as s:
# Grab new headers and cookies from login page
t = s.get("http://minewind.com/forums/ucp.php?mode=login", headers=headers)
sid = t.cookies['phpbb3_qpac2_sid'] # Store sid to be used in POST data.
# POST data to be sent
payload = {"login": "Login",
"password": "*********",
"redirect": "./ucp.php?mode=login",
"redirect": "index.php",
"sid": sid,
"username": "myusername"
}
# Send POST data to the login page, including proper headers.
s1 = s.post("http://minewind.com/forums/ucp.php?mode=login", data=payload, headers=t.headers)
print (t.headers)
# Check to see if we are really logged in, WHICH WE ARENT!!!! ;_;
s2 = s.get("http://minewind.com/forums/index.php", headers=t.headers)
# Pretty up the code and grab links.
perty = BeautifulSoup(s2.content)
perty.prettify()
for links in perty.find_all('a'):
print (links.get('href'))
I finally configured the POST data properly to my knowledge, but now i'm getting some weird connection errors, any ideas? ERRORS:
Traceback (most recent call last):
File "C:\Python33\lib\site-packages\requests\packages\urllib3\connectionpool.py", line 331, in _make_request
httplib_response = conn.getresponse(buffering=True)
TypeError: getresponse() got an unexpected keyword argument 'buffering'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Python33\lib\site-packages\requests\packages\urllib3\connectionpool.py", line 516, in urlopen
body=body, headers=headers)
File "C:\Python33\lib\site-packages\requests\packages\urllib3\connectionpool.py", line 333, in _make_request
httplib_response = conn.getresponse()
File "C:\Python33\lib\http\client.py", line 1143, in getresponse
response.begin()
File "C:\Python33\lib\http\client.py", line 354, in begin
version, status, reason = self._read_status()
File "C:\Python33\lib\http\client.py", line 316, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "C:\Python33\lib\socket.py", line 297, in readinto
return self._sock.recv_into(b)
ConnectionResetError: [WinError 10054] An existing connection was forcibly closed by the remote host
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Python33\lib\site-packages\requests\adapters.py", line 362, in send
timeout=timeout
File "C:\Python33\lib\site-packages\requests\packages\urllib3\connectionpool.py", line 559, in urlopen
_pool=self, _stacktrace=stacktrace)
File "C:\Python33\lib\site-packages\requests\packages\urllib3\util\retry.py", line 245, in increment
raise six.reraise(type(error), error, _stacktrace)
File "C:\Python33\lib\site-packages\requests\packages\urllib3\packages\six.py", line 309, in reraise
raise value.with_traceback(tb)
File "C:\Python33\lib\site-packages\requests\packages\urllib3\connectionpool.py", line 516, in urlopen
body=body, headers=headers)
File "C:\Python33\lib\site-packages\requests\packages\urllib3\connectionpool.py", line 333, in _make_request
httplib_response = conn.getresponse()
File "C:\Python33\lib\http\client.py", line 1143, in getresponse
response.begin()
File "C:\Python33\lib\http\client.py", line 354, in begin
version, status, reason = self._read_status()
File "C:\Python33\lib\http\client.py", line 316, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "C:\Python33\lib\socket.py", line 297, in readinto
return self._sock.recv_into(b)
requests.packages.urllib3.exceptions.ProtocolError: ('Connection aborted.', ConnectionResetError(10054, 'An existing con
nection was forcibly closed by the remote host', None, 10054))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\Anthony\site.py", line 28, in <module>
s2 = s.get("http://minewind.com/forums/index.php", headers=t.headers)
File "C:\Python33\lib\site-packages\requests\sessions.py", line 469, in get
return self.request('GET', url, **kwargs)
File "C:\Python33\lib\site-packages\requests\sessions.py", line 457, in request
resp = self.send(prep, **send_kwargs)
File "C:\Python33\lib\site-packages\requests\sessions.py", line 569, in send
r = adapter.send(request, **kwargs)
File "C:\Python33\lib\site-packages\requests\adapters.py", line 407, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', ConnectionResetError(10054, 'An existing connection was for
cibly closed by the remote host', None, 10054))
Any thoughts on why i'm getting these 'connection aborted' errors?
You can just use the user-agent headers for this instance, I complicating it by grabbing the login page headers which wasn't necessary. Also you don't need to know the sid cookie beforehand like I thought you did. You can just include it with the POST data as empty. Just make sure you are inspecting which form data is being passed with firebug or similar utilities stated above.
import requests
from bs4 import BeautifulSoup
import sys
headers={"User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36"}
with requests.Session() as s:
payload = {"login": "Login",
"password": "mypassword",
"redirect": "./ucp.php?mode=login",
"redirect": "index.php",
"sid": "",
"username": "myusername"}
url = "http://minewind.com/forums/index.php"
s1 = s.post("http://minewind.com/forums/ucp.php?mode=login", data=payload, headers=headers)
s2 = s.get(url, headers=headers)