I am trying to read the response after I have made a call to an API using python Twisted web client. I have made a POST call to the endpoint passing in a json structure, it should then return a status with either a message (if failed) or a json strucure if successful.
Using the code below I am able to see the message is getting called along with the status code, but I am not seeing the message/json structure.
The 'BeginningPrinter' is never getting called and I don't uderstand why.
Example of output:
$ python sample.py
Response version: (b'HTTP', 1, 0)
Response code: 401 | phrase : b'UNAUTHORIZED'
Response headers:
Response length: 28
Apologies that the code is so long, but I wanted to make sure it contains everything that I used to run it in it.
from io import BytesIO
import json
from twisted.internet import reactor
from twisted.web.client import Agent
from twisted.web.http_headers import Headers
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Protocol
from twisted.web.client import FileBodyProducer
agent = Agent(reactor)
class BeginningPrinter(Protocol):
def __init__(self, finished):
self.finished = finished
self.remaining = 1024 * 10
print('begin')
def dataReceived(self, bytes):
print('bytes')
if self.remaining:
display = bytes[:self.remaining]
print('Some data received:')
print(display)
self.remaining -= len(display)
def connectionLost(self, reason):
print('Finished receiving body:', reason.getErrorMessage())
self.finished.callback(None)
TESTDATA = { "keySequence": "2019-07-14" }
jsonData = json.dumps(TESTDATA)
body = BytesIO(jsonData.encode('utf-8'))
body = FileBodyProducer(body)
headerDict = \
{
'User-Agent': ['test'],
'Content-Type': ['application/json'],
'APIGUID' : ['ForTesting']
}
header = Headers(headerDict)
d = agent.request(b'POST', b' http://127.0.0.1:5000/receiveKeyCode', header, body)
def cbRequest(response):
print(f'Response version: {response.version}')
print(f'Response code: {response.code} | phrase : {response.phrase}')
print('Response headers:')
print('Response length:', response.length)
print(pformat(list(response.headers.getAllRawHeaders())))
print(response.deliverBody)
finished = Deferred()
response.deliverBody(BeginningPrinter(finished))
return finished
d.addCallback(cbRequest)
def cbShutdown(ignored):
#reactor.stop()
pass
d.addBoth(cbShutdown)
reactor.run()
You don't need all of that fluff code, if you are already using Flask then you can write to the API and get the values back in a few lines, If you are not then it makes sense to pip install it as it makes life a lot easier.
import json
import requests
headers = {
'content-type': 'application/json',
'APIGUID' : 'ForTesting'
}
conv = {"keySequence": "2019-07-14"}
s = json.dumps(conv)
res = requests.post("http://127.0.0.1:5000/receiveKeyCode",data=s, headers=headers)
print(res.text)
Reference: See this Stackoverflow link
Related
I have the following (with some strings modified) which works when using the requests library.
import requests
from pprint import pprint
import json
PEM = '/full/path/to/my.pem'
client_id='cliendID'
client_secret='clientSecret'
USER='myuser'
PSWD='mypwd'
url = 'https://theurl.com/request/'
data = {
"grant_type": "password",
"username": USER,
"password": PSWD
}
auth = (client_id, client_secret)
response = requests.post( url, auth=auth, data=data, verify=PEM )
answer = response.json()['answer']
print( answer )
The answer printed is what I expect.
The following usage of curl also works:
curl -iv --user cliendID:clientSecret --key /full/path/to/server.key --cert /full/path/to/my.pem -F grant_type=password -F username=myuser -F password=mypwd https://theurl.com/request/
However, when I try to do the same using Tornado and AsyncHTTPClient, I get a "Bad Request" response. Some sample Tornado code is:
from tornado.httpclient import AsyncHTTPClient
from tornado.ioloop import IOLoop
import json
async def get_content():
PEM = '/full/path/to/my.pem'
client_id='cliendID'
client_secret='clientSecret'
url = 'https://theurl.com/request/'
USER='myuser'
PSWD='mypwd'
data = {
"grant_type": "password",
"username": USER,
"password": PSWD
}
bodyData = json.dumps( data )
http_client = AsyncHTTPClient()
response = await http_client.fetch( url,
method = 'POST',
body = bodyData,
auth_username = client_id,
auth_password = client_secret,
ca_certs = PEM )
print( response.body.decode() )
async def main():
await get_content()
if __name__ == "__main__":
io_loop = IOLoop.current()
io_loop.run_sync(main)
If I had to guess, I believe the issue is with how I am sending the bodyData.
What do I need to change in the Tornado code so this will work...?
By default, the requests library and Tornado's AsyncHTTPClient, both, send the body data as form encoded data (i.e. with Content-Type: application/x-www-form-urlencoded).
The requests library automatically encodes the data dict with the correct content type.
But with Tornado's client, you're requried to manually encode the data as follows:
from urllib.parse import urlencode
bodyData = urlencode(data)
...
Cause of the error:
You're getting the Bad Request error because you're sending the data as JSON but the Content-Type header (sent automatically by Tornado) still says www-form-urlencoded. That means the server can't decode your supplied data because it doesn't know that it's JSON.
I'm trying to work with a third party API and I am having problems with sending the request when using the requests or even urllib.request.
Somehow when I use http.client I am successful sending and receiving the response I need.
To make life easier for me, I created an API class below:
class API:
def get_response_data(self, response: http.client.HTTPResponse) -> dict:
"""Get the response data."""
response_body = response.read()
response_data = json.loads(response_body.decode("utf-8"))
return response_data
The way I use it is like this:
api = API()
rest_api_host = "api.app.com"
connection = http.client.HTTPSConnection(rest_api_host)
token = "my_third_party_token"
data = {
"token":token
}
payload = json.loads(data)
headers = {
# some headers
}
connection.request("POST", "/some/endpoint/", payload, headers)
response = connection.getresponse()
response_data = api.get_response_data(response) # I get a dictionary response
This workflow works for me. Now I just want to write a test for the get_response_data method.
How do I instantiate a http.client.HTTPResponse with the desired output to be tested?
For example:
from . import API
from unittest import TestCase
class APITestCase(TestCase):
"""API test case."""
def setUp(self) -> None:
super().setUp()
api = API()
def test_get_response_data_returns_expected_response_data(self) -> None:
"""get_response_data() method returns expected response data in http.client.HTTPResponse"""
expected_response_data = {"token": "a_secret_token"}
# I want to do something like this
response = http.client.HTTPResponse(expected_response_data)
self.assertEqual(api.get_response_data(response), expected_response_data)
How can I do this?
From the http.client docs it says:
class http.client.HTTPResponse(sock, debuglevel=0, method=None, url=None)
Class whose instances are returned upon successful connection. Not instantiated directly by user.
I tried looking at socket for the sock argument in the instantiation but honestly, I don't understand it.
I tried reading the docs in
https://docs.python.org/3/library/http.client.html#http.client.HTTPResponse
https://docs.python.org/3/library/socket.html
Searched the internet on "how to test http.client.HTTPResponse" but I haven't found the answer I was looking for.
The long story short is I am working on building a server that serves as something as a chat bot. The server uses google dialog flow. Right now I have an endpoint exposed that allows me to talk to my server, when I hit that endpoint, google auth, as well as google dialog flow gets called. I am attempting to mock the response of dialog flow while leaving the actual server to respond to the network call. As of now my test looks like this.
This is my base test file:
import unittest
import mock
class BaseTest(unittest.TestCase, object):
def __init__(self, *args, **kwargs):
super(BaseTest, self).__init__(*args, *kwargs)
def auto_patch(self, patch_target):
patcher = mock.patch(patch_target)
patched = patcher.start()
self.addCleanup(patcher.stop)
return patched
This is my test file:
import json
import uuid
from os import path
from tests.base_test import BaseTest
from agent.api_service import app
import requests_mock
import pytest
from hamcrest import assert_that, has_items, equal_to
CWD = path.dirname(path.realpath(__file__))
class TestAudio(BaseTest):
def test__interact__full_no_stt(self):
payload = json.load(open("tests/json_payloads/test__interact__full_audio.json"))
u_session_id = str(uuid.uuid1())
payload["session_id"] = u_session_id
#mock a 500 back from STT
with open("tests/json_payloads/stt_500.json", "r") as issues_file:
mock_response = issues_file.read()
with requests_mock.Mocker() as m:
m.register_uri('POST', 'https://speech.googleapis.com/v1/speech:recognize', text=mock_response)
request, response = app.test_client.post("/agent_service/interact", data=json.dumps(payload))
self.assertEqual(200, response.status)
This is my google stt file:
import json
import requests
from agent.exceptions import GoogleSTTException
from agent.integrations.google.google_auth_service import get_auth_token
from agent.integrations.google.google_stt_request import GoogleSTTRequest
from agent.integrations.google.google_stt_response import GoogleSTTResponse
def speech_to_text(audio_string):
try:
google_stt_request = GoogleSTTRequest(audio_string).to_payload()
request_headers = dict()
request_headers['Authorization'] = 'Bearer ' + get_auth_token()
request_headers['Content-Type'] = 'application/json'
url = 'https://speech.googleapis.com/v1/speech:recognize'
google_response = requests.post(url, data=json.dumps(google_stt_request), headers=request_headers)
response = GoogleSTTResponse(google_response.json())
return response
except Exception as e:
raise GoogleSTTException('Received an error invoking google stt {}'.format(e))
Does anyone have any ideas on how I can mock the response from the google stt call, without touching the google auth call or the server call itself? I have tried a handful of things and so far no luck. I either end up mocking nothing, or both the google stt and auth call.
So I ended up moving away from the original implementation, but this is what got me there.
#responses.activate
def test__interact__full_no_stt(self):
payload = json.load(open("tests/json_payloads/test__interact__full_audio.json"))
u_session_id = str(uuid.uuid1())
payload["session_id"] = u_session_id
#mock a 500 back from STT
responses.add(responses.POST,
'https://speech.googleapis.com/v1/speech:recognize',
json={'error': 'broken'}, status=500)
request, response = app.test_client.post("/agent_service/interact", data=json.dumps(payload))
self.assertEqual(200, response.status)
result = response.json
Responses makes this much easier, just be sure to include the annotation at the top of the test.
Alright, so I'm a little outside of my league on this one I think.
I'm attempting to facilitate custom HTTP headers what is noted here:
API-Key = API key
API-Sign = Message signature using HMAC-SHA512 of (URI path + SHA256(nonce + POST data)) and base64 decoded secret API key
from https://www.kraken.com/help/api
I'm trying to work solely out of urllib if at all possible.
Below is one of many attempts to get it encoded like required:
import os
import sys
import time
import datetime
import urllib.request
import urllib.parse
import json
import hashlib
import hmac
import base64
APIKey = 'ThisKey'
secret = 'ThisSecret'
data = {}
data['nonce'] = int(time.time()*1000)
data['asset'] = 'ZUSD'
uripath = '/0/private/TradeBalance'
postdata = urllib.parse.urlencode(data)
encoded = (str(data['nonce']) + postdata).encode()
message = uripath.encode() + hashlib.sha256(encoded).digest()
signature = hmac.new(base64.b64decode(secret),
message, hashlib.sha512)
sigdigest = base64.b64encode(signature.digest())
#this is purely for checking how things are coming along.
print(sigdigest.decode())
headers = {
'API-Key': APIKey,
'API-Sign': sigdigest.decode()
}
The above may be working just fine, where I'm struggling now is appropriately getting it to the site.
This is my most recent attempt:
myBalance = urllib.urlopen('https://api.kraken.com/0/private/TradeBalance', urllib.parse.urlencode({'asset': 'ZUSD'}).encode("utf-8"), headers)
Any help is greatly appreciated.
Thanks!
urlopen doesn't support adding headers, so you need to create a Request object and pass it to urlopen:
url = 'https://api.kraken.com/0/private/TradeBalance'
body = urllib.parse.urlencode({'asset': 'ZUSD'}).encode("utf-8")
headers = {
'API-Key': APIKey,
'API-Sign': sigdigest.decode()
}
request = urllib.request.Request(url, data=body, headers=headers)
response = urllib.request.urlopen(request)
I'm trying to create a HTTPSConnection to this address: "android-review.googlesource.com" and send a json request.
This address: "android-review.googlesource.com" is for Gerrit code review system which uses REST API. You can find more information about the Gerrit Rest-api here:
https://gerrit-review.googlesource.com/Documentation/rest-api.html.
Each review in Gerrit code review system is related to a change request which I tried to get the change request information with a json request. This is the url and request:
url = "/gerrit_ui/rpc/ChangeDetailService"
req = {"jsonrpc" : "2.0",
"method": "changeDetail",
"params": [{"id": id}],
"id": 44
}
you can find the complete code here:
import socket, sys
import httplib
import pyodbc
import json
import types
import datetime
import urllib2
import os
import logging
import re, time
def GetRequestOrCached( url, method, data, filename):
path = os.path.join("json", filename)
if not os.path.exists(path):
data = MakeRequest(url, method, data)
time.sleep(1)
data = data.replace(")]}'", "")
f = open(path, "w")
f.write(data)
f.close()
return open(path).read()
def MakeRequest(url, method, data, port=443):
successful = False
while not successful:
try:
conn = httplib.HTTPSConnection("android-review.googlesource.com", port)
headers = {"Accept": "application/json,application/jsonrequest",
"Content-Type": "application/json; charset=UTF-8",
"Content-Length": len(data)}
conn.request(method, url, data, headers)
conn.set_debuglevel(1)
successful = True
except socket.error as err:
# this means a socket timeout
if err.errno != 10060:
raise(err)
else:
print err.errno, str(err)
print "sleep for 1 minute before retrying"
time.sleep(60)
resp = conn.getresponse()
if resp.status != 200:
raise GerritDataException("Got status code %d for request to %s" % (resp.status, url))
return resp.read()
#-------------------------------------------------
id=51750
filename = "%d-ChangeDetails.json" % id
url = "/gerrit_ui/rpc/ChangeDetailService"
req = {"jsonrpc" : "2.0",
"method": "changeDetail",
"params": [{"id": id}],
"id": 44
}
data = GetRequestOrCached(url, "POST", json.dumps(req), filename)
print json.loads(data)
In the code id means review id which can be a number between 1 and 51750, but not necessary all of these ids exist in the system so different numbers can be tried to see finally which one responds. For example these three ids definitely exist: 51750-51743-51742. I tried for these numbers but for all of them I got the same error:
"{"jsonrpc":"2.0","id":44,"error":{"code":-32603,"message":"No such service method"}}"
so I guess there is something wrong with code.
Why are you using url = "/gerrit_ui/rpc/ChangeDetailService"? That isn't in your linked REST documentation at all. I believe this is an older internal API which is no longer supported. I'm also not sure why your method is POST.
Instead, something like this works just fine for me:
curl "https://android-review.googlesource.com/changes/?q=51750"