I can't seem to Google it, but I want a function that does this:
Accept 3 arguments (or more, whatever):
URL
a dictionary of params
POST or GET
Return me the results, and the response code.
Is there a snippet that does this?
requests
https://github.com/kennethreitz/requests/
Here's a few common ways to use it:
import requests
url = 'https://...'
payload = {'key1': 'value1', 'key2': 'value2'}
# GET
r = requests.get(url)
# GET with params in URL
r = requests.get(url, params=payload)
# POST with form-encoded data
r = requests.post(url, data=payload)
# POST with JSON
import json
r = requests.post(url, data=json.dumps(payload))
# Response, status etc
r.text
r.status_code
httplib2
https://github.com/jcgregorio/httplib2
>>> from httplib2 import Http
>>> from urllib import urlencode
>>> h = Http()
>>> data = dict(name="Joe", comment="A test comment")
>>> resp, content = h.request("http://bitworking.org/news/223/Meet-Ares", "POST", urlencode(data))
>>> resp
{'status': '200', 'transfer-encoding': 'chunked', 'vary': 'Accept-Encoding,User-Agent',
'server': 'Apache', 'connection': 'close', 'date': 'Tue, 31 Jul 2007 15:29:52 GMT',
'content-type': 'text/html'}
Even easier: via the requests module.
import requests
get_response = requests.get(url='http://google.com')
post_data = {'username':'joeb', 'password':'foobar'}
# POST some form-encoded data:
post_response = requests.post(url='http://httpbin.org/post', data=post_data)
To send data that is not form-encoded, send it serialised as a string (example taken from the documentation):
import json
post_response = requests.post(url='http://httpbin.org/post', data=json.dumps(post_data))
# If using requests v2.4.2 or later, pass the dict via the json parameter and it will be encoded directly:
post_response = requests.post(url='http://httpbin.org/post', json=post_data)
You could use this to wrap urllib2:
def URLRequest(url, params, method="GET"):
if method == "POST":
return urllib2.Request(url, data=urllib.urlencode(params))
else:
return urllib2.Request(url + "?" + urllib.urlencode(params))
That will return a Request object that has result data and response codes.
import urllib
def fetch_thing(url, params, method):
params = urllib.urlencode(params)
if method=='POST':
f = urllib.urlopen(url, params)
else:
f = urllib.urlopen(url+'?'+params)
return (f.read(), f.code)
content, response_code = fetch_thing(
'http://google.com/',
{'spam': 1, 'eggs': 2, 'bacon': 0},
'GET'
)
[Update]
Some of these answers are old. Today I would use the requests module like the answer by robaple.
I know you asked for GET and POST but I will provide CRUD since others may need this just in case: (this was tested in Python 3.7)
#!/usr/bin/env python3
import http.client
import json
print("\n GET example")
conn = http.client.HTTPSConnection("httpbin.org")
conn.request("GET", "/get")
response = conn.getresponse()
data = response.read().decode('utf-8')
print(response.status, response.reason)
print(data)
print("\n POST example")
conn = http.client.HTTPSConnection('httpbin.org')
headers = {'Content-type': 'application/json'}
post_body = {'text': 'testing post'}
json_data = json.dumps(post_body)
conn.request('POST', '/post', json_data, headers)
response = conn.getresponse()
print(response.read().decode())
print(response.status, response.reason)
print("\n PUT example ")
conn = http.client.HTTPSConnection('httpbin.org')
headers = {'Content-type': 'application/json'}
post_body ={'text': 'testing put'}
json_data = json.dumps(post_body)
conn.request('PUT', '/put', json_data, headers)
response = conn.getresponse()
print(response.read().decode(), response.reason)
print(response.status, response.reason)
print("\n delete example")
conn = http.client.HTTPSConnection('httpbin.org')
headers = {'Content-type': 'application/json'}
post_body ={'text': 'testing delete'}
json_data = json.dumps(post_body)
conn.request('DELETE', '/delete', json_data, headers)
response = conn.getresponse()
print(response.read().decode(), response.reason)
print(response.status, response.reason)
Related
I want to send POST request to VNF to save Services .
Here is my code .
class APIClient:
def __init__(self, api_type, auth=None):
if api_type == 'EXTERNAL':
self.auth_client = auth
def api_call(self, http_verb, base_url, api_endpoint, headers=None, request_body=None, params=None):
if headers is not None:
headers = merge_dicts(headers, auth_header)
else:
headers = auth_header
url_endpoint = base_url + api_endpoint
request_body = json.dumps(request_body)
if http_verb == 'POST':
api_resp = requests.post(url_endpoint, data=request_body, headers=headers)
return api_resp
else:
return False
def add_service():
for service in service_list:
dict = service.view_service()
auth_dict = {
'server_url': 'https://authserver.nisha.com/auth/',
'client_id': 'vnf_api',
'realm_name': 'nisha,
'client_secret_key': 'abcd12345',
'grant_type': 'client_credentials'
}
api_client = APIClient(api_type='EXTERNAL', auth=auth_dict)
response = api_client.api_call(http_verb='POST',
base_url='http://0.0.0.0:5054',
api_endpoint='/vnf/service-management/v1/services',
request_body=f'{dict}')
print(response)
if response.ok:
print("success")
else:
print("no")
When I run this code it prints
<Response [415]>
no
All the functions in VNF side are working without issues and I have no issue with GET services api call.
How to fix this ?
If you need to post application/json data to an endpoint, you need to use the json kwarg in requests.post rather than the data kwarg.
To show the difference between the json and data kwargs in requests.post:
import requests
from requests import Request
# This is form-encoded
r = Request('POST', 'https://myurl.com', headers={'hello': 'world'}, data={'some': 'data'})
x = r.prepare()
x.headers
# note the content-type here
{'hello': 'world', 'Content-Length': '9', 'Content-Type': 'application/x-www-form-urlencoded'}
# This is json content
r = Request('POST', 'https://myurl.com', headers={'hello': 'world'}, json={'some': 'data'})
x = r.prepare()
x.headers
{'hello': 'world', 'Content-Length': '16', 'Content-Type': 'application/json'}
So you don't need the json.dumps step here at all:
url_endpoint = base_url + api_endpoint
if http_verb == 'POST':
api_resp = requests.post(url_endpoint, json=request_body, headers=headers)
I'm trying to send a HTTP POST request with Python. I can get it to work with 3.0, but I couldn't find a good example on 2.7.
hdr = {"content-type": "application/json"}
payload= ("<html><body><h1>Sorry it's not Friday yet</h1> </body></html>")
r = requests.post("http://my-url/api/Html", json={"HTML": payload})
with open ('c:/temp/a.pdf', 'wb') as f:
b64str = json.loads(r.text)['BinaryData'] #base 64 string is in BinaryData attr
binStr = binascii.a2b_base64(b64str) #convert base64 string to binary
f.write(binStr)
The api takes a json in this format:
{
HTML : "a html string"
}
and returns a json in this format:
{
BinaryData: 'base64 encoded string'
}
In Python 2.x it should be like this
import json
import httplib
body =("<html><body><h1>Sorry it's not Friday yet</h1> </body></html>")
payload = {'HTML' : body}
hdr = {"content-type": "application/json"}
conn = httplib.HTTPConnection('my-url')
conn.request('POST', '/api/Html', json.dumps(payload), hdr)
response = conn.getresponse()
data = response.read() # same as r.text in 3.x
The standard way is with the urllib2 module
from urllib import urlencode
import urllib2
def http_post(url, data):
post = urlencode(data)
req = urllib2.Request(url, post)
response = urllib2.urlopen(req)
return response.read()
I have a couple of web services calls using the request package in python one is purely form and WORKS:
r = requests.post('http://localhost:5000/coordinator/finished-crawl', \
data = {'competitorId':value})
And the other uses JSON and does not work:
service_url = 'http://localhost:5000/coordinator/save-page'
data = {'Url': url, 'CompetitorId': competitorID, \
'Fetched': self.generateTimestamp(), 'Html': html}
headers = {'Content-type': 'application/json'}
r = requests.post(service_url, data=json.dumps(data), headers=headers)
Now if do not include headers I use the headers as above, I get a 404, but if I do not include as
r = requests.post(service_url, data=json.dumps(data))
I get a 415. I have tried looking at other post on stackoverflow and from what I can tell the call is correct. I have tested the web service via the application postman and it works. Can some tell me what is wrong or point me in the right direction?
THE FULL METHOD
def saveContent(self, url, competitorID, html):
temp = self.cleanseHtml(html)
service_url = 'http://localhost:5000/coordinator/save-page'
data = {'Url': url, 'CompetitorId': competitorID, \
'Fetched': self.generateTimestamp(), \
'Html': temp}
headers = {'Content-type': 'application/json'}
r = requests.post(service_url, json=json.dumps(data), headers=headers)
r = requests.post(service_url, json=json.dumps(data))
And cleanseHTML:
def cleanseHtml(self, html):
return html.replace("\\", "\\\\")\
.replace("\"", "\\\"")\
.replace("\n", "")\
.replace("\r", "")
I'm trying to extract the cookies from a HTTPResponse using cookielib.CookieJar.extract_cookies(), but I keep getting an error saying that the response object doesn't have an .info attribute. I know it's more designed for pseudo-file objects like those returned by urllib2.urlopen, but what's the canonical way to extract cookies from a HTTPResponse? Here's what I've got:
def _make_request(self, loc, headers, data=None, retry=True):
retries = 0
max_retries = self._retry_max if retry else 1
self._request = urllib2.Request('http://example.com/')
self._connection = httplib.HTTPSConnection(host)
try:
while retries < max_retries:
try:
self._request.add_data(data)
self._connection.request(self._request.get_method(), self._request.get_selector() + loc,
self._request.get_data(), headers)
resp = self._connection.getresponse()
self._cookies.extract_cookies(resp, self._request) # problems!
if len(self._cookies) > 0:
# do something
...
Thanks
Try to use key set-cookie to get cookies. Here is example for you.
#!/usr/bin/env python
import urllib
import httplib2
http = httplib2.Http()
url = 'http://www.example.com/login'
body = {'USERNAME': 'foo', 'PASSWORD': 'bar'}
headers = {'Content-type': 'application/x-www-form-urlencoded'}
response, content = http.request(url, 'POST', headers=headers, body=urllib.urlencode(body))
headers = {'Cookie': response['set-cookie']}
url = 'http://www.example.com/home'
response, content = http.request(url, 'GET', headers=headers)
This little baby:
import urllib2
import simplejson as json
opener = urllib2.build_opener()
opener.addheaders.append(('Content-Type', 'application/json'))
response = opener.open('http://localhost:8000',json.dumps({'a': 'b'}))
Produces the following request (as seen with ngrep):
sudo ngrep -q -d lo '^POST .* localhost:8000'
T 127.0.0.1:51668 -> 127.0.0.1:8000 [AP]
POST / HTTP/1.1..Accept-Encoding: identity..Content-Length: 10..Host: localhost:8000..Content-Type: application/x-www-form-urlencoded..Connection: close..User-Agent:
Python-urllib/2.7....{"a": "b"}
I do not want that Content-Type: application/x-www-form-urlencoded. I am explicitely saying that I want ('Content-Type', 'application/json')
What's going on here?!
If you want to set custom headers you should use a Request object:
import urllib2
import simplejson as json
opener = urllib2.build_opener()
req = urllib2.Request('http://localhost:8000', data=json.dumps({'a': 'b'}),
headers={'Content-Type': 'application/json'})
response = opener.open(req)
I got hit by the same stuff and came up with this little gem:
import urllib2
import simplejson as json
class ChangeTypeProcessor(BaseHandler):
def http_request(self, req):
req.unredirected_hdrs["Content-type"] = "application/json"
return req
opener = urllib2.build_opener()
self.opener.add_handler(ChangeTypeProcessor())
response = opener.open('http://localhost:8000',json.dumps({'a': 'b'}))
You just add a handler for HTTP requests that replaces the header that OpenerDirector previously added.
Python version:Python 2.7.15
I found that in urllib2.py:1145:
for name, value in self.parent.addheaders:
name = name.capitalize()
if not request.has_header(name):
request.add_unredirected_header(name, value)
...
def has_header(self, header_name):
return (header_name in self.headers or
header_name in self.unredirected_hdrs)
Otherwise,application/x-www-form-urlencoded has been in unredirected_hdrs and it won't be overwrite
You can solve like
import urllib.request
from http.cookiejar import CookieJar
import json
url = 'http://www.baidu.com'
req_dict = {'k': 'v'}
cj = CookieJar()
handler = urllib.request.HTTPCookieProcessor(cj)
opener = urllib.request.build_opener(handler)
req_json = json.dumps(req_dict)
req_post = req_json.encode('utf-8')
headers = {}
#headers['Content-Type'] = 'application/json'
req = urllib.request.Request(url=url, data=req_post, headers=headers)
#urllib.request.install_opener(opener)
#res = urllib.request.urlopen(req)
# or
res = opener.open(req)
res = res.read().decode('utf-8')
The problem is the capitalization of the Header name. You should use Content-type and not Content-Type.