Python Requests POST url encoded request not working - python

I am trying to make a post request to a url with Requests in Python. I am getting 500 ERROR message. I think the problem is in not properly encoding in content-type to x-www-form-urlencoded. If I try the same request in Postman it is working.
My Python code:
import requests
def get_pdf_link_by_post(url, payload):
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
r = requests.post(url, data=payload, headers=headers)
print requests
print "**************Headers*******************"
print r.headers
print r.content
def process_kat_causelist(date, adv_name):
url = 'http://kat.kar.nic.in:8080/causelistsearch.jsp'
for court_hall_idx in range(1, 4):
court_hall = str(court_hall_idx)+'~Court Hall '+str(court_hall_idx)
for list_idx in range(1, 7):
list_num = str(list_idx)+'~List '+str(list_idx)
payload = {'firstClDate': date, 'secondCldate': date, 'courthall': court_hall, 'lists': list_num}
print payload
get_pdf_link_by_post(url, payload)
process_kat_causelist('25/08/2015', 'ani')
Postman request:


In the payload secondCldate must be secondClDate

Related

401 with a POST using requests

Using Postman, I can make a POST to an endpoint with a JSON body, and it returns be some results.
No authorisation on the endpoint.
Trying to get this to work with requests fails with a 401.
What am I missing here:
Code:
import json
import requests
url = 'https://wabi-australia-southeast-api.analysis.windows.net/public/reports/querydata?synchronous=true'
header = {"content-type": "application/json"}
body = {"version":"1.0.0","queries":[{"Query":{"Commands":[{"SemanticQueryDataShapeCommand":{"Query":{"Version":2,"From":[{"Name":"d1","Entity":"dimLGA","Type":0},{"Name":"l","Entity":"Linelist","Type":0}],"Select":[{"Column":{"Expression":{"SourceRef":{"Source":"d1"}},"Property":"LGAName"},"Name":"dimLGA.LGAName"},{"Measure":{"Expression":{"SourceRef":{"Source":"l"}},"Property":"Cases"},"Name":"Linelist.Cases"}],"Where":[{"Condition":{"Not":{"Expression":{"Comparison":{"ComparisonKind":0,"Left":{"Column":{"Expression":{"SourceRef":{"Source":"d1"}},"Property":"LGAName"}},"Right":{"Literal":{"Value":"null"}}}}}}},{"Condition":{"In":{"Expressions":[{"Column":{"Expression":{"SourceRef":{"Source":"l"}},"Property":"clin_status_n"}}],"Values":[[{"Literal":{"Value":"'Admitted to ICU'"}}],[{"Literal":{"Value":"'Admitted, not known to be in ICU'"}}],[{"Literal":{"Value":"'Home isolation'"}}],[{"Literal":{"Value":"'Hotel detention'"}}],[{"Literal":{"Value":"'Hospital in the home'"}}],[{"Literal":{"Value":"'Under investigation'"}}]]}}}]},"Binding":{"Primary":{"Groupings":[{"Projections":[0,1]}]},"DataReduction":{"DataVolume":3,"Primary":{"Window":{"Count":500}}},"Version":1}}}]},"CacheKey":"{\"Commands\":[{\"SemanticQueryDataShapeCommand\":{\"Query\":{\"Version\":2,\"From\":[{\"Name\":\"d1\",\"Entity\":\"dimLGA\",\"Type\":0},{\"Name\":\"l\",\"Entity\":\"Linelist\",\"Type\":0}],\"Select\":[{\"Column\":{\"Expression\":{\"SourceRef\":{\"Source\":\"d1\"}},\"Property\":\"LGAName\"},\"Name\":\"dimLGA.LGAName\"},{\"Measure\":{\"Expression\":{\"SourceRef\":{\"Source\":\"l\"}},\"Property\":\"Cases\"},\"Name\":\"Linelist.Cases\"}],\"Where\":[{\"Condition\":{\"Not\":{\"Expression\":{\"Comparison\":{\"ComparisonKind\":0,\"Left\":{\"Column\":{\"Expression\":{\"SourceRef\":{\"Source\":\"d1\"}},\"Property\":\"LGAName\"}},\"Right\":{\"Literal\":{\"Value\":\"null\"}}}}}}},{\"Condition\":{\"In\":{\"Expressions\":[{\"Column\":{\"Expression\":{\"SourceRef\":{\"Source\":\"l\"}},\"Property\":\"clin_status_n\"}}],\"Values\":[[{\"Literal\":{\"Value\":\"'Admitted to ICU'\"}}],[{\"Literal\":{\"Value\":\"'Admitted, not known to be in ICU'\"}}],[{\"Literal\":{\"Value\":\"'Home isolation'\"}}],[{\"Literal\":{\"Value\":\"'Hotel detention'\"}}],[{\"Literal\":{\"Value\":\"'Hospital in the home'\"}}],[{\"Literal\":{\"Value\":\"'Under investigation'\"}}]]}}}]},\"Binding\":{\"Primary\":{\"Groupings\":[{\"Projections\":[0,1]}]},\"DataReduction\":{\"DataVolume\":3,\"Primary\":{\"Window\":{\"Count\":500}}},\"Version\":1}}}]}","QueryId":"","ApplicationContext":{"DatasetId":"5b547437-24c9-4b22-92de-900b3b3f4785","Sources":[{"ReportId":"964ef513-8ff4-407c-8068-ade1e7f64ca5"}]}}],"cancelQueries":[],"modelId":1959902}
r = requests.post(url, data=json.dumps(body), headers=header)
print(r.status_code)
print(r.raise_for_status())
This returns a 401:
HTTPError: 401 Client Error: Unauthorized for url:
https://wabi-australia-southeast-api.analysis.windows.net/public/reports/querydata?synchronous=true

Just found out Postman lets you generate a python code snippet for Requests. Amazing.
This solved my problem.
import requests
url = "https://wabi-australia-southeast-api.analysis.windows.net/public/reports/querydata"
payload = "{\"version\":\"1.0.0\",\"queries\":[{\"Query\":{\"Commands\":[{\"SemanticQueryDataShapeCommand\":{\"Query\":{\"Version\":2,\"From\":[{\"Name\":\"d1\",\"Entity\":\"dimLGA\",\"Type\":0},{\"Name\":\"l\",\"Entity\":\"Linelist\",\"Type\":0}],\"Select\":[{\"Column\":{\"Expression\":{\"SourceRef\":{\"Source\":\"d1\"}},\"Property\":\"LGAName\"},\"Name\":\"dimLGA.LGAName\"},{\"Measure\":{\"Expression\":{\"SourceRef\":{\"Source\":\"l\"}},\"Property\":\"Cases\"},\"Name\":\"Linelist.Cases\"}],\"Where\":[{\"Condition\":{\"Not\":{\"Expression\":{\"Comparison\":{\"ComparisonKind\":0,\"Left\":{\"Column\":{\"Expression\":{\"SourceRef\":{\"Source\":\"d1\"}},\"Property\":\"LGAName\"}},\"Right\":{\"Literal\":{\"Value\":\"null\"}}}}}}},{\"Condition\":{\"In\":{\"Expressions\":[{\"Column\":{\"Expression\":{\"SourceRef\":{\"Source\":\"l\"}},\"Property\":\"clin_status_n\"}}],\"Values\":[[{\"Literal\":{\"Value\":\"'Admitted to ICU'\"}}],[{\"Literal\":{\"Value\":\"'Admitted, not known to be in ICU'\"}}],[{\"Literal\":{\"Value\":\"'Home isolation'\"}}],[{\"Literal\":{\"Value\":\"'Hotel detention'\"}}],[{\"Literal\":{\"Value\":\"'Hospital in the home'\"}}],[{\"Literal\":{\"Value\":\"'Under investigation'\"}}]]}}}]},\"Binding\":{\"Primary\":{\"Groupings\":[{\"Projections\":[0,1]}]},\"DataReduction\":{\"DataVolume\":3,\"Primary\":{\"Window\":{\"Count\":500}}},\"Version\":1}}}]},\"CacheKey\":\"{\\\"Commands\\\":[{\\\"SemanticQueryDataShapeCommand\\\":{\\\"Query\\\":{\\\"Version\\\":2,\\\"From\\\":[{\\\"Name\\\":\\\"d1\\\",\\\"Entity\\\":\\\"dimLGA\\\",\\\"Type\\\":0},{\\\"Name\\\":\\\"l\\\",\\\"Entity\\\":\\\"Linelist\\\",\\\"Type\\\":0}],\\\"Select\\\":[{\\\"Column\\\":{\\\"Expression\\\":{\\\"SourceRef\\\":{\\\"Source\\\":\\\"d1\\\"}},\\\"Property\\\":\\\"LGAName\\\"},\\\"Name\\\":\\\"dimLGA.LGAName\\\"},{\\\"Measure\\\":{\\\"Expression\\\":{\\\"SourceRef\\\":{\\\"Source\\\":\\\"l\\\"}},\\\"Property\\\":\\\"Cases\\\"},\\\"Name\\\":\\\"Linelist.Cases\\\"}],\\\"Where\\\":[{\\\"Condition\\\":{\\\"Not\\\":{\\\"Expression\\\":{\\\"Comparison\\\":{\\\"ComparisonKind\\\":0,\\\"Left\\\":{\\\"Column\\\":{\\\"Expression\\\":{\\\"SourceRef\\\":{\\\"Source\\\":\\\"d1\\\"}},\\\"Property\\\":\\\"LGAName\\\"}},\\\"Right\\\":{\\\"Literal\\\":{\\\"Value\\\":\\\"null\\\"}}}}}}},{\\\"Condition\\\":{\\\"In\\\":{\\\"Expressions\\\":[{\\\"Column\\\":{\\\"Expression\\\":{\\\"SourceRef\\\":{\\\"Source\\\":\\\"l\\\"}},\\\"Property\\\":\\\"clin_status_n\\\"}}],\\\"Values\\\":[[{\\\"Literal\\\":{\\\"Value\\\":\\\"'Admitted to ICU'\\\"}}],[{\\\"Literal\\\":{\\\"Value\\\":\\\"'Admitted, not known to be in ICU'\\\"}}],[{\\\"Literal\\\":{\\\"Value\\\":\\\"'Home isolation'\\\"}}],[{\\\"Literal\\\":{\\\"Value\\\":\\\"'Hotel detention'\\\"}}],[{\\\"Literal\\\":{\\\"Value\\\":\\\"'Hospital in the home'\\\"}}],[{\\\"Literal\\\":{\\\"Value\\\":\\\"'Under investigation'\\\"}}]]}}}]},\\\"Binding\\\":{\\\"Primary\\\":{\\\"Groupings\\\":[{\\\"Projections\\\":[0,1]}]},\\\"DataReduction\\\":{\\\"DataVolume\\\":3,\\\"Primary\\\":{\\\"Window\\\":{\\\"Count\\\":500}}},\\\"Version\\\":1}}}]}\",\"QueryId\":\"\",\"ApplicationContext\":{\"DatasetId\":\"5b547437-24c9-4b22-92de-900b3b3f4785\",\"Sources\":[{\"ReportId\":\"964ef513-8ff4-407c-8068-ade1e7f64ca5\"}]}}],\"cancelQueries\":[],\"modelId\":1959902}"
headers = {
'Content-Type': 'text/plain'
}
response = requests.request("POST", url, headers=headers, data = payload)
print(response.text.encode('utf8'))

Scraping this site

Im trying to mimic this POST request from this site with this payload:
from this URL: https://surviv.io/stats/gert1
Here is an image of the request im trying to mimic.
Here is my current code in python:
import requests
headers = {'content-type': 'application/json; charset=UTF-8'}
url = 'https://surviv.io/api/user_stats'
payload = {"slug":"gert1","interval":"all","mapIdFilter":"-1"}
r = requests.post(url=url, headers=headers, data=payload)
print(r.content)
This returns:
b'<html>\r\n<head><title>500 Internal Server Error</title></head>\r\n<body bgcolor="white">\r\n<center><h1>500 Internal Server Error</h1></center>\r\n<hr><center>nginx</center>\r\n</body>\r\n</html>\r\n'
This is not what I want it return. I want it to return the exact response shown in the response tab of the user_stats requests, which contains the player's stats.
This is what I want it to return:
{"slug":"gert1","username":"GERT","player_icon":"","banned":false,"wins":61,"kills":2830,"games":2034,"kpg":"1.4","modes":[{"teamMode":1,"games":1512,"wins":46,"kills":2230,"winPct":"3.0","mostKills":21,"mostDamage":1872,"kpg":"1.5","avgDamage":169,"avgTimeAlive":92},{"teamMode":2,"games":255,"wins":4,"kills":234,"winPct":"1.6","mostKills":8,"mostDamage":861,"kpg":"0.9","avgDamage":162,"avgTimeAlive":102},{"teamMode":4,"games":267,"wins":11,"kills":366,"winPct":"4.1","mostKills":17,"mostDamage":2225,"kpg":"1.4","avgDamage":246,"avgTimeAlive":125}]}

You should use the json attribute rather than data in the post method. r = requests.post(url=url, headers=headers, json=payload)

Change your code to following your forgot to use json :
import json
import requests
headers = {'content-type': 'application/json; charset=UTF-8'}
url = 'https://surviv.io/api/user_stats'
payload = {"slug":"gert1","interval":"all","mapIdFilter":"-1"}
r = requests.post(url=url, headers=headers, data=json.dumps(payload))
print(r.content)

Receiving a JSON response from a POST request

Using a POST request, my goal is to receive an authorization code from a JSON response. However, the only response I'm getting is the HTML of the webpage I'm connecting to, not the desired JSON response.
import requests
from base64 import b64encode
appAuth = b64encode(b"QT8txxxxxxxx:n76mxxxxxxxxx").decode("ascii")
headers = { 'Authorization' : 'Basic %s' % appAuth }
url = "http://demo.skubana.com?grant_type=authorization_code&redirect_uri=demo.skubana.com/appstore&code=LCqYHU&cid=Y29tcGFueUlkMTI0MDYw"
r = requests.post(url, headers=headers,json={})
print(r.status_code)
print(r.content)

The supplied API documentation for Skubana is, in my opinion, shockingly under-specified.
What you may have missed is this part:
Add the request path: /oauth/token
which, to be fair, is not reflected in their 'sample URL':
Sample Url: https://blah.skubana.com?grant_type=authorization_code&redirect_uri=redirect.com/callback&code=ABCDE&cid=ABCD1234
When you change your URL to include that path, I get a 401 Unauthorized response instead:
>>> url = "http://demo.skubana.com/oauth/token?grant_type=authorization_code&redirect_uri=demo.skubana.com/appstore&code=LCqYHU&cid=Y29tcGFueUlkMTI0MDYw"
>>> r = requests.post(url, headers=headers)
>>> r.status_code
401
This too, incidentally, also involves a redirect from POST to GET. But I strongly suspect that with the right credentials (which I don't have) you'd actually get your token.
When you post to https://demo.skubana.com, you are redirected, first to the same URL to make a GET request, with:
Location: https://demo.skubana.com/?grant_type=authorization_code&redirect_uri=demo.skubana.com/appstore&code=LCqYHU&cid=Y29tcGFueUlkMTI0MDYw
then on to your dashboard:
Location: /work/dashboard
and finally to the login page:
Location: https://demo.skubana.com/login
All this is recorded in the r.history list:
>>> r.history
[<Response [302]>, <Response [302]>, <Response [302]>]
Note that you don't have to set json={}, just use:
r = requests.post(url, headers=headers)
The requests library can take care of the Base64 encoding for you as that's just part of the Basic Authentication scheme, just pass in the YOUR_APP_KEY and YOUR_APP_SECRET values as username and password:
YOUR_APP_KEY = "QT8txxxxxxxx"
YOUR_APP_SECRET = "n76mxxxxxxxxx"
url = "http://demo.skubana.com/oauth/token"
parameters = {
"grant_type": "authorization_code",
"redirect_uri": "demo.skubana.com/appstore",
"code": "LCqYHU",
"cid": "Y29tcGFueUlkMTI0MDYw",
}
r = requests.post(url, params=parameters, auth=(YOUR_APP_KEY, YOUR_APP_SECRET))
Finally, given the redirect and complete lack of POST parameters, I strongly suspect that a GET request works just as well:
r = requests.get(url, params=parameters, auth=(YOUR_APP_KEY, YOUR_APP_SECRET))
This just bypasses the POST -> GET redirect.
Normally, a OAuth2 request for a token like this would post the parameters in the POST body, so do also try:
r = requests.post(url, data=parameters, auth=(YOUR_APP_KEY, YOUR_APP_SECRET))
if the params=parameters options don't get you your token, still.

I have some troubles with a post request

I'm trying to scrape a website but i get a 500 response.
It's a relocation website and i'm trying to get prices for different input parameters (volume, destination, departureCity ...).
import requests
from bs4 import BeautifulSoup
url = "https://www.demenagerfacile.com/devis/create"
payload = "volume_set_with_vcalc=false&id_vc_quote=&mode=MOVE_CLASSIC&fromCity=150+Rue+Saint-Maur%2C+Paris%2C+France&locationDeparture=%7B%22lat%22%3A%2248.8690238%22%2C%22lng%22%3A%222.3745715999999675%22%7D&placeIdDeparture=%22ChIJOTRebuRt5kcRoklIrmMZx6Y%22&countryDep=FR&toCity=Rue+de+Marseille%2C+Lyon%2C+France&locationDestination=%7B%22lat%22%3A%2245.7509119%22%2C%22lng%22%3A%224.83963289999997%22%7D&placeIdDestination=%22Eh5SdWUgZGUgTWFyc2VpbGxlLCBMeW9uLCBGcmFuY2UiLiosChQKEgm7NI_oRer0RxEcVzzVtXmrQxIUChIJl4foalHq9EcR8CG75CqrCAQ%22&countryDest=FR&fixedDate=on&date=2019-07-18&volume=30"
headers = {
"content-type": "application/x-www-form-urlencoded",
"cookie": "PHPSESSID=k3f99mg9urimu9dpc1ot7vudf5; __stripe_mid=e5c971f7-5e78-4e61-92db-a62dda48fd2c; _ga=GA1.2.110393239.1561101566; _gid=GA1.2.1597485519.1561101566; previousServerId=0; _fbp=fb.1.1561101566214.588023040; _hjIncludedInSample=1; hubspotutk=e7e9f524c880e1091d415cfc5cbd4903; __hssrc=1; apzContact=%5B%7B%22buttonIdKey%22%3A%229deacb07173c79d76baf4f58c56037ad%22%2C%22apizeeSessionId%22%3A%22334d6232-930f-41ec-aab9-0753c7b3036a%22%2C%22apiconfSession%22%3A%229deacb07173c79d76baf4f58c56037ad%22%2C%22apiconfOfflineMessageState%22%3Anull%7D%5D; cookiebanner-accepted=1; __stripe_sid=5e2fb6df-ea5c-470a-98f3-44e172955633; __hstc=78993675.e7e9f524c880e1091d415cfc5cbd4903.1561101566944.1561101566944.1561106755827.2; _gat_UA-65262710-1=1; __hssc=78993675.3.1561106755827; apiCCId=334d6232-930f-41ec-aab9-0753c7b3036a-0000; apiKey=d7897a4712a9413c3a363e737954d853; sessionId=230a5a70-9401-11e9-935a-19ca5a0a60c1",
}
response = requests.request("POST", url, data=payload)
print(response)

Requests’ simple API means that all forms of HTTP request are as obvious. For example, this is how you make an HTTP POST request
Replace your code
response = requests.request("POST", url, data=payload)
TO
response = requests.post(url,headers=headers, data=payload)

Hi #mhabak1 I just checked and it seems to be working,
Please add headers while posting the request like below,
response = requests.post(url, data=payload, headers=headers)
Thanks

Python 2.6 json error 400 and 401 when i trying to fetch from Ubiquity Unify API

I am trying to generate a voucher from Ubiquity UNIFY web portal.
It has a API, and there is some ideas to use this with PHP, but I would like to use Python 2.6 to generate.. (I can ofc ourse use other Python versions if this is a must)
My code is:
import urllib, json
import urllib2
import unifi
import os
import requests
def JsonLogin():
payload = {"username" : "myuser","password" : "mypassword"}
r = requests.post("https://ubnt.myserver.com:8443/api/login", data=payload, verify=False)
print r
def JsonApi():
payload = {"cmd":"create-voucher","minutes":1440,"n":1}
r = requests.post("https://ubnt.myserver.com:8443/api/cmd/hotspot", data=payload, verify=False)
print r
JsonLogin();
JsonApi();
My Result is:
<Response [400]>
<Response [401]>
Is there any logic why it returns 400, witch means error...
Is this because I sent invalid JSON or is this because the syntax or JSON tags are wrong?

try this
payload= '{"username" : "myuser","password" : "mypassword", "strict":true}'
r = requests.post("https://ubnt.myserver.com:8443/api/login", data=payload, verify=False)
print r
It should return 200
Then set someway the cookie value that the login returns into a variable
cookies=r.cookies
then
r = requests.post("https://ubnt.myserver.com:8443/api/cmd/hotspot", data=payload, verify=False, cookies=cookies)
I used it with GET and it works

Categories