I'm trying to extract the cookies from a HTTPResponse using cookielib.CookieJar.extract_cookies(), but I keep getting an error saying that the response object doesn't have an .info attribute. I know it's more designed for pseudo-file objects like those returned by urllib2.urlopen, but what's the canonical way to extract cookies from a HTTPResponse? Here's what I've got:
def _make_request(self, loc, headers, data=None, retry=True):
retries = 0
max_retries = self._retry_max if retry else 1
self._request = urllib2.Request('http://example.com/')
self._connection = httplib.HTTPSConnection(host)
try:
while retries < max_retries:
try:
self._request.add_data(data)
self._connection.request(self._request.get_method(), self._request.get_selector() + loc,
self._request.get_data(), headers)
resp = self._connection.getresponse()
self._cookies.extract_cookies(resp, self._request) # problems!
if len(self._cookies) > 0:
# do something
...
Thanks
Try to use key set-cookie to get cookies. Here is example for you.
#!/usr/bin/env python
import urllib
import httplib2
http = httplib2.Http()
url = 'http://www.example.com/login'
body = {'USERNAME': 'foo', 'PASSWORD': 'bar'}
headers = {'Content-type': 'application/x-www-form-urlencoded'}
response, content = http.request(url, 'POST', headers=headers, body=urllib.urlencode(body))
headers = {'Cookie': response['set-cookie']}
url = 'http://www.example.com/home'
response, content = http.request(url, 'GET', headers=headers)
Related
I want to send POST request to VNF to save Services .
Here is my code .
class APIClient:
def __init__(self, api_type, auth=None):
if api_type == 'EXTERNAL':
self.auth_client = auth
def api_call(self, http_verb, base_url, api_endpoint, headers=None, request_body=None, params=None):
if headers is not None:
headers = merge_dicts(headers, auth_header)
else:
headers = auth_header
url_endpoint = base_url + api_endpoint
request_body = json.dumps(request_body)
if http_verb == 'POST':
api_resp = requests.post(url_endpoint, data=request_body, headers=headers)
return api_resp
else:
return False
def add_service():
for service in service_list:
dict = service.view_service()
auth_dict = {
'server_url': 'https://authserver.nisha.com/auth/',
'client_id': 'vnf_api',
'realm_name': 'nisha,
'client_secret_key': 'abcd12345',
'grant_type': 'client_credentials'
}
api_client = APIClient(api_type='EXTERNAL', auth=auth_dict)
response = api_client.api_call(http_verb='POST',
base_url='http://0.0.0.0:5054',
api_endpoint='/vnf/service-management/v1/services',
request_body=f'{dict}')
print(response)
if response.ok:
print("success")
else:
print("no")
When I run this code it prints
<Response [415]>
no
All the functions in VNF side are working without issues and I have no issue with GET services api call.
How to fix this ?
If you need to post application/json data to an endpoint, you need to use the json kwarg in requests.post rather than the data kwarg.
To show the difference between the json and data kwargs in requests.post:
import requests
from requests import Request
# This is form-encoded
r = Request('POST', 'https://myurl.com', headers={'hello': 'world'}, data={'some': 'data'})
x = r.prepare()
x.headers
# note the content-type here
{'hello': 'world', 'Content-Length': '9', 'Content-Type': 'application/x-www-form-urlencoded'}
# This is json content
r = Request('POST', 'https://myurl.com', headers={'hello': 'world'}, json={'some': 'data'})
x = r.prepare()
x.headers
{'hello': 'world', 'Content-Length': '16', 'Content-Type': 'application/json'}
So you don't need the json.dumps step here at all:
url_endpoint = base_url + api_endpoint
if http_verb == 'POST':
api_resp = requests.post(url_endpoint, json=request_body, headers=headers)
I have a couple of web services calls using the request package in python one is purely form and WORKS:
r = requests.post('http://localhost:5000/coordinator/finished-crawl', \
data = {'competitorId':value})
And the other uses JSON and does not work:
service_url = 'http://localhost:5000/coordinator/save-page'
data = {'Url': url, 'CompetitorId': competitorID, \
'Fetched': self.generateTimestamp(), 'Html': html}
headers = {'Content-type': 'application/json'}
r = requests.post(service_url, data=json.dumps(data), headers=headers)
Now if do not include headers I use the headers as above, I get a 404, but if I do not include as
r = requests.post(service_url, data=json.dumps(data))
I get a 415. I have tried looking at other post on stackoverflow and from what I can tell the call is correct. I have tested the web service via the application postman and it works. Can some tell me what is wrong or point me in the right direction?
THE FULL METHOD
def saveContent(self, url, competitorID, html):
temp = self.cleanseHtml(html)
service_url = 'http://localhost:5000/coordinator/save-page'
data = {'Url': url, 'CompetitorId': competitorID, \
'Fetched': self.generateTimestamp(), \
'Html': temp}
headers = {'Content-type': 'application/json'}
r = requests.post(service_url, json=json.dumps(data), headers=headers)
r = requests.post(service_url, json=json.dumps(data))
And cleanseHTML:
def cleanseHtml(self, html):
return html.replace("\\", "\\\\")\
.replace("\"", "\\\"")\
.replace("\n", "")\
.replace("\r", "")
I am trying to convert the following Java code to Python. Not sure what I am doing wrong, but I end up with an internal server error 500 with python.
Is the "body" in httplib.httpConnection method equivalent to Java httpentity?
Any other thoughts on what could be wrong?
The input information I collect is correct for sure.
Any help will be really appreciated. I have tried several things, but end up with the same internal server error.
Java Code:
HttpEntity reqEntitiy = new StringEntity("loginTicket="+ticket);
HttpRequestBase request = reMethod.getRequest(uri, reqEntitiy);
request.addHeader("ticket", ticket);
HttpResponse response = httpclient.execute(request);
HttpEntity responseEntity = response.getEntity();
StatusLine responseStatus = response.getStatusLine();
Python code:
url = serverURL + "resources/slmservices/templates/"+templateId+"/options"
#Create the request
ticket = ticket.replace("'",'"')
headers = {"ticket":ticket}
print "ticket",ticket
reqEntity = "loginTicket="+ticket
body = "loginTicket="+ticket
url2 = urlparse.urlparse(serverURL)
h1 = httplib.HTTPConnection(url2.hostname,8580)
print "h1",h1
url3 = urlparse.urlparse(url)
print "url path",url3.path
ubody = {"loginTicket":ticket}
data = urllib.urlencode(ubody)
conn = h1.request("GET",url3.path,data,headers)
#conn = h1.request("GET",url3.path)
response = h1.getresponse()
lines = response.read()
print "response.status",response.status
print "response.reason",response.reason
You don't need to go this low level. Using urllib2 instead:
import urllib2
from urllib import urlencode
url = "{}resources/slmservices/templates/{}/options".format(
serverURL, templateId)
headers = {"ticket": ticket}
params = {"loginTicket": ticket}
url = '{}?{}'.format(url, urlencode(params))
request = urllib2.Request(url, headers=headers)
response = urllib2.urlopen(request)
print 'Status', response.getcode()
print 'Response data', response.read()
Note that the parameters are added to the URL to form URL query parameters.
You can do this simpler still by installing the requests library:
import requests
url = "{}resources/slmservices/templates/{}/options".format(
serverURL, templateId)
headers = {"ticket": ticket}
params = {"loginTicket": ticket}
response = requests.get(url, params=params, headers=headers)
print 'Status', response.status
print 'Response data', response.content # or response.text for Unicode
Here requests takes care of URL-encoding the URL query string parameters and adding it to the URL for you, just like Java does.
This little baby:
import urllib2
import simplejson as json
opener = urllib2.build_opener()
opener.addheaders.append(('Content-Type', 'application/json'))
response = opener.open('http://localhost:8000',json.dumps({'a': 'b'}))
Produces the following request (as seen with ngrep):
sudo ngrep -q -d lo '^POST .* localhost:8000'
T 127.0.0.1:51668 -> 127.0.0.1:8000 [AP]
POST / HTTP/1.1..Accept-Encoding: identity..Content-Length: 10..Host: localhost:8000..Content-Type: application/x-www-form-urlencoded..Connection: close..User-Agent:
Python-urllib/2.7....{"a": "b"}
I do not want that Content-Type: application/x-www-form-urlencoded. I am explicitely saying that I want ('Content-Type', 'application/json')
What's going on here?!
If you want to set custom headers you should use a Request object:
import urllib2
import simplejson as json
opener = urllib2.build_opener()
req = urllib2.Request('http://localhost:8000', data=json.dumps({'a': 'b'}),
headers={'Content-Type': 'application/json'})
response = opener.open(req)
I got hit by the same stuff and came up with this little gem:
import urllib2
import simplejson as json
class ChangeTypeProcessor(BaseHandler):
def http_request(self, req):
req.unredirected_hdrs["Content-type"] = "application/json"
return req
opener = urllib2.build_opener()
self.opener.add_handler(ChangeTypeProcessor())
response = opener.open('http://localhost:8000',json.dumps({'a': 'b'}))
You just add a handler for HTTP requests that replaces the header that OpenerDirector previously added.
Python version:Python 2.7.15
I found that in urllib2.py:1145:
for name, value in self.parent.addheaders:
name = name.capitalize()
if not request.has_header(name):
request.add_unredirected_header(name, value)
...
def has_header(self, header_name):
return (header_name in self.headers or
header_name in self.unredirected_hdrs)
Otherwise,application/x-www-form-urlencoded has been in unredirected_hdrs and it won't be overwrite
You can solve like
import urllib.request
from http.cookiejar import CookieJar
import json
url = 'http://www.baidu.com'
req_dict = {'k': 'v'}
cj = CookieJar()
handler = urllib.request.HTTPCookieProcessor(cj)
opener = urllib.request.build_opener(handler)
req_json = json.dumps(req_dict)
req_post = req_json.encode('utf-8')
headers = {}
#headers['Content-Type'] = 'application/json'
req = urllib.request.Request(url=url, data=req_post, headers=headers)
#urllib.request.install_opener(opener)
#res = urllib.request.urlopen(req)
# or
res = opener.open(req)
res = res.read().decode('utf-8')
The problem is the capitalization of the Header name. You should use Content-type and not Content-Type.
I can't seem to Google it, but I want a function that does this:
Accept 3 arguments (or more, whatever):
URL
a dictionary of params
POST or GET
Return me the results, and the response code.
Is there a snippet that does this?
requests
https://github.com/kennethreitz/requests/
Here's a few common ways to use it:
import requests
url = 'https://...'
payload = {'key1': 'value1', 'key2': 'value2'}
# GET
r = requests.get(url)
# GET with params in URL
r = requests.get(url, params=payload)
# POST with form-encoded data
r = requests.post(url, data=payload)
# POST with JSON
import json
r = requests.post(url, data=json.dumps(payload))
# Response, status etc
r.text
r.status_code
httplib2
https://github.com/jcgregorio/httplib2
>>> from httplib2 import Http
>>> from urllib import urlencode
>>> h = Http()
>>> data = dict(name="Joe", comment="A test comment")
>>> resp, content = h.request("http://bitworking.org/news/223/Meet-Ares", "POST", urlencode(data))
>>> resp
{'status': '200', 'transfer-encoding': 'chunked', 'vary': 'Accept-Encoding,User-Agent',
'server': 'Apache', 'connection': 'close', 'date': 'Tue, 31 Jul 2007 15:29:52 GMT',
'content-type': 'text/html'}
Even easier: via the requests module.
import requests
get_response = requests.get(url='http://google.com')
post_data = {'username':'joeb', 'password':'foobar'}
# POST some form-encoded data:
post_response = requests.post(url='http://httpbin.org/post', data=post_data)
To send data that is not form-encoded, send it serialised as a string (example taken from the documentation):
import json
post_response = requests.post(url='http://httpbin.org/post', data=json.dumps(post_data))
# If using requests v2.4.2 or later, pass the dict via the json parameter and it will be encoded directly:
post_response = requests.post(url='http://httpbin.org/post', json=post_data)
You could use this to wrap urllib2:
def URLRequest(url, params, method="GET"):
if method == "POST":
return urllib2.Request(url, data=urllib.urlencode(params))
else:
return urllib2.Request(url + "?" + urllib.urlencode(params))
That will return a Request object that has result data and response codes.
import urllib
def fetch_thing(url, params, method):
params = urllib.urlencode(params)
if method=='POST':
f = urllib.urlopen(url, params)
else:
f = urllib.urlopen(url+'?'+params)
return (f.read(), f.code)
content, response_code = fetch_thing(
'http://google.com/',
{'spam': 1, 'eggs': 2, 'bacon': 0},
'GET'
)
[Update]
Some of these answers are old. Today I would use the requests module like the answer by robaple.
I know you asked for GET and POST but I will provide CRUD since others may need this just in case: (this was tested in Python 3.7)
#!/usr/bin/env python3
import http.client
import json
print("\n GET example")
conn = http.client.HTTPSConnection("httpbin.org")
conn.request("GET", "/get")
response = conn.getresponse()
data = response.read().decode('utf-8')
print(response.status, response.reason)
print(data)
print("\n POST example")
conn = http.client.HTTPSConnection('httpbin.org')
headers = {'Content-type': 'application/json'}
post_body = {'text': 'testing post'}
json_data = json.dumps(post_body)
conn.request('POST', '/post', json_data, headers)
response = conn.getresponse()
print(response.read().decode())
print(response.status, response.reason)
print("\n PUT example ")
conn = http.client.HTTPSConnection('httpbin.org')
headers = {'Content-type': 'application/json'}
post_body ={'text': 'testing put'}
json_data = json.dumps(post_body)
conn.request('PUT', '/put', json_data, headers)
response = conn.getresponse()
print(response.read().decode(), response.reason)
print(response.status, response.reason)
print("\n delete example")
conn = http.client.HTTPSConnection('httpbin.org')
headers = {'Content-type': 'application/json'}
post_body ={'text': 'testing delete'}
json_data = json.dumps(post_body)
conn.request('DELETE', '/delete', json_data, headers)
response = conn.getresponse()
print(response.read().decode(), response.reason)
print(response.status, response.reason)