I have method which uses requests to download content from the url in a document. Below is the test case
#patch.object(Session, 'get')
def test_download_success(mock_req_get):
inputDocument = {"contentUri": "http://test"}
def res():
r = requests.Response()
r.status_code = 200
r.text = "test"
return r
mock_req_get.return_value = res()
output = download(inputDocument)
assert output["rawContent"] == "test"
I am trying to mock the requests session and the get return value but the unittest is not working and it shows an exception at the line
mock_req_get.return_value = res()
def download(document):
queryConnectTimeoutSecond = 5
queryReadTimeoutSecond = 5
url = document['contentUri']
adapter = HTTPAdapter(max_retries=3)
session = requests.Session()
session.mount(url, adapter)
try:
result = session.get(url, timeout=(queryConnectTimeoutSecond, queryReadTimeoutSecond))
document["rawContent"] = result.text
except Exception as err:
logger.error(f"An error ocurred: {err}")
document["error"] = {
"code": 500,
"message": "Internal Server Error",
"details": ["Document Crawler Failed"]
}
return document
I am a complete noob in python programming. Trying to understand if this is the right way to write unittests.
Related
I am having trouble understanding how mocking works when the get responses involve using the with keyword. Here is an example I am following for my class `Album' and I have been successful when I am mocking a url as seen below:
def find_album_by_id(id):
url = f'https://jsonplaceholder.typicode.com/albums/{id}'
response = requests.get(url)
if response.status_code == 200:
return response.json()['title']
else:
return None
Here the test
class TestAlbum(unittest.TestCase):
#patch('album.requests')
def test_find_album_by_id_success(self, mock_requests):
# mock the response
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
'userId': 1,
'id': 1,
'title': 'hello',
}
# specify the return value of the get() method
mock_requests.get.return_value = mock_response
# call the find_album_by_id and test if the title is 'hello'
self.assertEqual(find_album_by_id(1), 'hello')
However, I am trying to understand how this would work with the with keyword involved in the code logic which I am using in my project. This is how I changed the method
def find_album_by_id(id):
url = f'https://jsonplaceholder.typicode.com/albums/{id}'
with requests.get(url) as response:
pipelines = response.json()
if response.status_code == 200:
return pipelines['title']
else:
return None
Here is my current test:
#patch('album.requests.get')
def test_find_album_by_id_success(self, mock_requests):
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.pipelines.response.json = {
'userId': 1,
'id': 1,
'title': 'hello',
}
mock_requests.return_value.json.return_value = mock_response
self.assertEqual(find_album_by_id(1), 'hello')
Thanks
I have tried debugging the test and it just never receives the status code of 200 so I am not sure I am mocking response correctly at all? From my understanding the mock_response is supposed to have that status code of 200 but breakline indicates that response is just a MagicMock with nothing in it.
I am using python and I apologize as this is probably a very simple concept that I am not grasping but I am not very familiar with the flask framework. Right now I'm making a server with flask but run into a 405 error every time I try to do something with the twitter api shortly after. Right now my code looks something like this
In a separate file:
from flask import Flask
from threading import Thread
app = Flask('')
#app.route('/', methods=['GET', 'POST', 'DELETE'])
def home():
return "Hello world!"
def run():
app.run(host='0.0.0.0',port=8080)
app.run(debug=True, host="0.0.0.0")
def keep_alive():
t = Thread(target=run)
t.start()
Then I call my code in the main file:
from flaskfile import flask_file
flask_file()
MEDIA_ENDPOINT_URL = 'https://upload.twitter.com/1.1/media/upload.json'
POST_TWEET_URL = 'https://api.twitter.com/1.1/statuses/update.json'
CONSUMER_KEY = consumerkey
CONSUMER_SECRET = secret
ACCESS_TOKEN = accesstoken
ACCESS_TOKEN_SECRET = tokensecret
#fileName = check_and_download()
VIDEO_FILENAME = (filename)
oauth = OAuth1(CONSUMER_KEY,
client_secret=CONSUMER_SECRET,
resource_owner_key=ACCESS_TOKEN,
resource_owner_secret=ACCESS_TOKEN_SECRET)
class VideoTweet(object):
def __init__(self, file_name):
'''
Defines video tweet properties
'''
self.video_filename = file_name
self.total_bytes = os.path.getsize(self.video_filename)
self.media_id = None
self.processing_info = None
def upload_init(self):
'''
Initializes Upload
'''
print('INIT')
request_data = {
'command': 'INIT',
'media_type': 'video/mp4',
'total_bytes': self.total_bytes,
'media_category': 'tweet_video'
}
req = requests.post(url=MEDIA_ENDPOINT_URL, data=request_data, auth=oauth)
media_id = req.json()['media_id']
self.media_id = media_id
print('Media ID: %s' % str(media_id))
def upload_append(self):
'''
Uploads media in chunks and appends to chunks uploaded
'''
segment_id = 0
bytes_sent = 0
file = open(self.video_filename, 'rb')
while bytes_sent < self.total_bytes:
chunk = file.read(4*1024*1024)
print('APPEND')
request_data = {
'command': 'APPEND',
'media_id': self.media_id,
'segment_index': segment_id
}
files = {
'media':chunk
}
req = requests.post(url=MEDIA_ENDPOINT_URL, data=request_data, files=files, auth=oauth)
if req.status_code < 200 or req.status_code > 299:
print(req.status_code)
print(req.text)
sys.exit(0)
segment_id = segment_id + 1
bytes_sent = file.tell()
print('%s of %s bytes uploaded' % (str(bytes_sent), str(self.total_bytes)))
print('Upload chunks complete.')
def upload_finalize(self):
'''
Finalizes uploads and starts video processing
'''
print('FINALIZE')
request_data = {
'command': 'FINALIZE',
'media_id': self.media_id
}
req = requests.post(url=MEDIA_ENDPOINT_URL, data=request_data, auth=oauth)
print(req.json())
self.processing_info = req.json().get('processing_info', None)
self.check_status()
def check_status(self):
'''
Checks video processing status
'''
if self.processing_info is None:
return
state = self.processing_info['state']
print('Media processing status is %s ' % state)
if state == u'succeeded':
return
if state == u'failed':
sys.exit(0)
check_after_secs = self.processing_info['check_after_secs']
print('Checking after %s seconds' % str(check_after_secs))
time.sleep(check_after_secs)
print('STATUS')
request_params = {
'command': 'STATUS',
'media_id': self.media_id
}
req = requests.get(url=MEDIA_ENDPOINT_URL, params=request_params, auth=oauth)
self.processing_info = req.json().get('processing_info', None)
self.check_status()
def tweet(self):
'''
Publishes Tweet with attached video
'''
request_data = {
#leave status blank
'status': '',
'media_ids': self.media_id
}
req = requests.post(url=POST_TWEET_URL, data=request_data, auth=oauth)
print(req.json())
if __name__ == '__main__':
videoTweet = VideoTweet(VIDEO_FILENAME)
videoTweet.upload_init()
videoTweet.upload_append()
videoTweet.upload_finalize()
videoTweet.tweet()
The error returned states the following:
"Error response
Error code: 405
Message: Method Not Allowed.
Error code explanation: 405 - Specified method is invalid for this resource."
Basically I call the flask file then use twitter api to upload a file, but for some reason using the twitter api always results in a 405 error and my flask server is no longer accessible. How can I adjust my flask file to allow me to upload using the api?
405 errors are thrown when the wrong request method is used. For example a POST request being sent when a GET request was expected, or a GET request being sent when a POST request was expected.
In your check_status function you have
req = requests.get(url=MEDIA_ENDPOINT_URL, ...
But according to Twitter's documentation you should be using a post request here
req = requests.post(url=MEDIA_ENDPOINT_URL, ...
I have a template for a Python function that calls an API and returns data. I'd like to run load testing against that API using locust.
import requests
proxies = {"http" : None,
"https" : None}
verify = "/data/certs/abc123.crt"
def call_api(par1, par2, par3):
r = requests.post(url = 'https://ABCD123.XYZ.QWERTY:9010/public/api/v1/ABC_TEST/query',
json = {"par1" : par1, "par2" : par2, "par3" : par3}, verify = verify, proxies = proxies)
return r
How could I translate this into a locust class?
I do it this way:
# prepare the headers to send to the remote host
headers = {"key":"value",
"key":"value"}
# prepare the data for the POST body
data = {"key":"value",
"key":"value"}```
with connection_object.client.post(url, headers=headers,
json=data, name=task_name, catch_response=True) as response:
# convert the response to JSON
msg = json.loads(response.content.decode('utf-8'))
if response.status_code == 200:
# we got a 200 OK
response.success()
else:
response.failure("failure text")
In this example, this code runs in a separate function called from the UserBehavior class, so connection_object is "self" if you are doing this inside the task in the class.
In your example though, change
r = requests.post(url = 'https://ABCD123.XYZ.QWERTY:9010/public/api/v1/ABC_TEST/query',
json = {"par1" : par1, "par2" : par2, "par3" : par3}, verify = verify, proxies = proxies)
to
class UserBehavior(SequentialTaskSet):
#task()
def task1(self):
r = self.client.post('https://ABCD123.XYZ.QWERTY:9010/public/api/v1/ABC_TEST/query',
json = {"par1" : par1, "par2" : par2, "par3" : par3},
verify = verify, proxies = proxies)
I am sure you have seen this, which shows "get", but not "post". If you are unfamiliar with the requests library, it can get confusing.
Hope that helps!
I am trying to pass my session object from one class to another. But I am not sure whats happening.
class CreateSession:
def __init__(self, user, pwd, url="http://url_to_hit"):
self.url = url
self.user = user
self.pwd = pwd
def get_session(self):
sess = requests.Session()
r = sess.get(self.url + "/", auth=(self.user, self.pwd))
print(r.content)
return sess
class TestGet(CreateSession):
def get_response(self):
s = self.get_session()
print(s)
data = s.get(self.url + '/some-get')
print(data.status_code)
print(data)
if __name__ == "__main__":
TestGet(user='user', pwd='pwd').get_response()
I am getting 401 for get_response(). Not able to understand this.
What's a 401?
The response you're getting means that you're unauthorised to access the resource.
A session is used in order to persist headers and other prerequisites throughout requests, why are you creating the session every time rather than storing it in a variable?
As is, the session should work the only issue is that you're trying to call a resource that you don't have access to. - You're not passing the url parameter either in the initialisation.
Example of how you can effectively use Session:
from requests import Session
from requests.exceptions import HTTPError
class TestGet:
__session = None
__username = None
__password = None
def __init__(self, username, password):
self.__username = username
self.__password = password
#property
def session(self):
if self.__session is None:
self.__session = Session()
self.__session.auth = (self.__user, self.__pwd)
return self.__session
#session.setter
def session(self, value):
raise AttributeError('Setting \'session\' attribute is prohibited.')
def get_response(self, url):
try:
response = self.session.get(url)
# raises if the status code is an error - 4xx, 5xx
response.raise_for_status()
return response
except HTTPError as e:
# you received an http error .. handle it here (e contains the request and response)
pass
test_get = TestGet('my_user', 'my_pass')
first_response = test_get.get_response('http://your-website-with-basic-auth.com')
second_response = test_get.get_response('http://another-url.com')
my_session = test_get.session
my_session.get('http://url.com')
I am using requests 2.12.1 library in Python 2.7 and Django 1.10 for consume web services. When I use a session for save cookies and use persistence, and pass 10 seconds ~ without use any web service, my view regenerates the object requests.Session()...
This makes web service doesn't serve me, because my view has changed the cookies.
This is my Views.py:
client_session = requests.Session()
#watch_login
def loginUI(request):
response = client_session.post(URL_API+'login/', data={'username': username, 'password': password,})
json_login = response.json()
#login_required(login_url="/login/")
def home(request):
response_statistics = client_session.get(URL_API+'statistics/')
log('ClientSession: '+str(client_session))
try:
json_statistics = response_statistics.json()
except ValueError:
log('ExcepcionClientSession: '+str(client_session))
return logoutUI(request)
return render(request, "UI/home.html", {
'phone_calls' : json_statistics['phone_calls'],
'mobile_calls' : json_statistics['mobile_calls'],
'other_calls' : json_statistics['other_calls'],
'top_called_phones' : json_statistics['top_called_phones'],
'call_ranges_week' : json_statistics['call_ranges_week'],
'call_ranges_weekend' : json_statistics['call_ranges_weekend'],
'access_data' : accessData(request.user.username),
})
def userFeaturesFormInit(clientRequest):
log('FeaturesClientSession: '+str(client_session))
response = client_session.get(URL_API+'features/')
try:
json_features = response.json()
except ValueError as e:
log('ExcepcionFeaturesClientSession: '+str(client_session))
raise e
Thank you.
I fixed it specifying cookies manually, and saving it in the request.
client_session = requests.Session()
response = client_session.post(URL_API+'login/', {'username': username, 'password': password,})
request.session['cookiecsrf'] = client_session.cookies['csrftoken']
request.session['cookiesession'] = client_session.cookies['sessionid']
And sending it in the gets/posts:
cookies = {'csrftoken' : request.session['cookiecsrf'], 'sessionid': request.session['cookiesession']}
response = requests.get(URL, cookies=cookies)