I'm writing a unittest class to ensure a method tests for a success, and then tests for an Exception. I'm passing a response that should trigger the exception, but in the testing method it does not get raised. Of note, I can manually make the exception raise in the actual method.
Test class:
class TestAPI(TestCase):
def test_send_method(self):
with mock.patch('requests.post') as mock_request:
mock_response = mock.Mock()
mock_response.json.return_value = {
"success": "true"
}
mock_request.return_value = mock_response
send_method() // THIS WORKS NICELY
# Test that errors from the API are handled correctly.
with self.assertRaises(SendException):
mock_response.status_code = 500
mock_response.json.return_value = {
'errors': 'An error has occurred.',
}
send_method() // THIS RAISES NO EXCEPTION
As I said, It's odd because I can manually trigger the 500 status code in the actual method and it raises fine. I can even change the initial mock response success to err and it will raise in the actual method. Why would it not raise in the unittest?
Method being tested:
class SendException(Exception):
pass
def send_method():
session_headers = {
"Content-Type": "application/json",
}
session_body = {
"send": "yes"
}
session_url = u'{}'.format(URL)
session_response = requests.post(session_url, json=session_body, headers=session_headers)
try:
if(session_response.json().get('errors') is not None):
raise SendException(
'Returned error with status {}: {}'.format(
session_response.status_code,
session_response.json().get('errors')
)
)
except ValueError as err:
raise SendException(
'Responded with non-json and status code {}. Error: {} - Response Text: {}'.format(
session_response.status_code,
err,
session_response.text
)
)
Related
call to http()URl & download the file in S3 bucket. its working. then in 2nd part i am calling guardduty & give location of s3 file to create threat intel set. while running code i am getting below error:-
Response
{
"errorMessage": "'BadRequestException' object has no attribute 'message'",
"errorType": "AttributeError",
"requestId": "bec541eb-a315-4f65-9fa9-3f1139e31f86",
"stackTrace": [
" File \"/var/task/lambda_function.py\", line 38, in lambda_handler\n if \"name already exists\" in error.message:\n"
]
}
i want to create threat intel set using the file which is in S3--(downloaded from the URl)
code:-
import boto3
from datetime import datetime
import requests.packages.urllib3 as urllib3
def lambda_handler(event, context):
url='https://rules.emergingthreats.net/blockrules/compromised-ips.txt' # put your url here
bucket = 'awssaflitetifeeds-security' #your s3 bucket
key = 'GDfeeds/compromised-ips.csv' #your desired s3 path or filename
s3=boto3.client('s3')
http=urllib3.PoolManager()
s3.upload_fileobj(http.request('GET', url,preload_content=False), bucket, key)
#------------------------------------------------------------------
# Guard Duty
#------------------------------------------------------------------
location = "https://s3://awssaflitetifeeds-security/GDfeeds/compromised-ips.csv"
timeStamp = datetime.now()
name = "TF-%s"%timeStamp.strftime("%Y%m%d")
guardduty = boto3.client('guardduty')
response = guardduty.list_detectors()
if len(response['DetectorIds']) == 0:
raise Exception('Failed to read GuardDuty info. Please check if the service is activated')
detectorId = response['DetectorIds'][0]
try:
response = guardduty.create_threat_intel_set(
Activate=True,
DetectorId=detectorId,
Format='FIRE_EYE',
Location=location,
Name=name
)
except Exception as error:
if "name already exists" in error.message:
found = False
response = guardduty.list_threat_intel_sets(DetectorId=detectorId)
for setId in response['ThreatIntelSetIds']:
response = guardduty.get_threat_intel_set(DetectorId=detectorId, ThreatIntelSetId=setId)
if (name == response['Name']):
found = True
response = guardduty.update_threat_intel_set(
Activate=True,
DetectorId=detectorId,
Location=location,
Name=name,
ThreatIntelSetId=setId
)
break
if not found:
raise
#-------------------------------------------------------------------
# Update result data
#------------------------------------------------------------------
result = {
'statusCode': '200',
'body': {'message': "You requested: %s day(s) of /view/iocs indicators in CSV"%environ['DAYS_REQUESTED']}
}
except Exception as error:
logging.getLogger().error(str(error))
responseStatus = 'FAILED'
reason = error.message
result = {
'statusCode': '500',
'body': {'message': error.message}
}
finally:
#------------------------------------------------------------------
# Send Result
#------------------------------------------------------------------
if 'ResponseURL' in event:
send_response(event, context, responseStatus, responseData, event['LogicalResourceId'], reason)
The reason you are getting that error message is because the exception being returned from guardduty.create_threat_intel_set does not have the message attribute directly on the exception. I think you want either error.response['Message'] or error.response['Error']['Message'] for this exception case.
A couple of other suggestions:
you should replace the except Exception which is matching the exception showing an already-existing name with something more targeted. I'd recommend looking at what exceptions the guardduty client can throw for the particular operation and catch just the one you care about.
it is likely better to check that error.response['Error']['Code'] is exactly the error you want rather than doing a partial string match.
I can not get the exception to throw on this unit test.
def test_something(monkeypatch):
# Arrange
os.environ["ENTITY"] = "JURKAT" # should monkeypatch this, but ignore for now.
opgave_or_sag_id = "S7777777"
url = "https://testdriven.io"
auth = ("test", "test")
tx = 999999999
patch_called = False
def mock_get(*args, **kwargs):
nonlocal patch_called
patch_called = True
return MockResponseGet410()
monkeypatch.setattr(Session, "get", mock_get)
# Act
with pytest.raises(Exception) as exc_info:
dsu.fetch_entity_from_green(opgave_or_sag_id, url, auth, tx)
# Assert
assert patch_called
assert exc_info.value.args[0] == "Expected status code 200 (or 410 with invalid opgaver), but got 410 for entity JURKAT. Error: Invalid opgave - It's gone - https://testdriven.io"
class MockResponseGet410:
def __init__(self):
self.status_code = 410
self.text = "the opgave has status: INVALID, it's gone now."
self.reason = "It's gone"
self.url = "https://testdriven.io"
self.headers = {
"Content-Type": "application/json",
"Content-Length": "123",
"Content-Location": "tx=123",
}
# From dsu
def fetch_entity_from_green(opgave_or_sag_id, url, auth, tx):
"""Retrieve missing entity from green's api.
Parameters
----------
opgave_or_sag_id : string, required
tx : int, required
A tx number.
url : str, required
auth : AWSAuthObject, required
Returns
-------
entity : dict
A dict representing the entity from green.
status_code : int
"""
try:
ENTITY = os.environ["ENTITY"]
url_with_id = url + str(opgave_or_sag_id)
s = fetch_request_session_with_retries()
r = s.get(url_with_id, auth=auth)
handle_non_200_response_for_invalid_opgaver(r, opgave_or_sag_id, ENTITY)
# I deleted the rest, not relevant in this test as the above function should throw an exception.
except Exception as e:
print(f"An exception occured on request with id {opgave_or_sag_id}: {e}")
The exception should be thrown in handle_non_200_response_for_invalid_opgaver because mock_get returns a 410 status code and ENTITY is set to JURKAT:
def handle_non_200_response_for_invalid_opgaver(request, opgave_or_sag_id, ENTITY):
"""
Handles a non-200 response from the API but allows 410 responses on invalid opgaver.
"""
# 410 because Team Green returns this for invalid opgaver, which becomes a valid response.
if request.status_code != 200 and (
request.status_code == 410 and ENTITY != "OPGAVER"
):
print(f"Status code for request on {opgave_or_sag_id}: {request.status_code}")
raise Exception( # TODO be more explicit with exception.
f"Expected status code 200 (or 410 with invalid opgaver), but got {request.status_code} for entity {ENTITY}. Error: {request.text} - {request.reason} - {request.url}"
)
I can get an exception to throw using pytest.raises(Exception) in a different test (see below), and the test passes, so I'm on the right track:
def test_handle_non_200_response():
# Arrange
r = MockResponse()
# Act
with pytest.raises(Exception) as exc_info:
handle_non_200_response(r)
# Assert
assert (
exc_info.value.args[0]
== "Expected status code 200, but got 504. Error: Gateway Timeout - Exceeded 30 seconds - https://testdriven.io"
)
class MockResponse:
def __init__(self):
self.status_code = 504
self.text = "Gateway Timeout"
self.reason = "Exceeded 30 seconds"
self.url = "https://testdriven.io"
def json(self):
return {"id": 1}
def handle_non_200_response(request):
"""
Handles a non-200 response from the API.
"""
if request.status_code != 200:
print(f"Status code for request on {id}: {request.status_code}")
raise Exception(
f"Expected status code 200, but got {request.status_code}. Error: {request.text} - {request.reason} - {request.url}"
)
Can you see where I have gone astray?
I would like to write a unit test case for HTTPError exception part based on the error response content I get. But I have now idea how I can mock the response so that the unit test can reach doSomething1() instead of doSomething2().
foo.py
def get_result_from_API():
#Try to call API here...
def getSomething():
try:
result = get_result_from_API()
except HTTPError as error:
error_json = error.response.json()
if error_json.get("error").get("code") == "00001":
doSomething1()
else:
doSomething2()
raise error
Unit Test
#patch('foo.doSomething2')
#patch('foo.doSomething1')
#patch('foo.get_result_from_API')
def testGetSomething(get_result_from_API,doSomething1,doSomething2):
mock_response = Mock()
mock_response.return_value = {
"error":{
"code": "00001",
"message": "error message for foo reason"
}
}
get_result_from_API.side_effect = HTTPError(response=mock_response)
with self.assertRaises(HTTPError):
foo.getSomething()
doSomething1.assert_called_once()
The current result is that doSomething1() is not called where as doSomething2() is called.
Since the getSomething function calls:
error_json = error.response.json()
the solution is to mock the call to .json()
mock_response.json.return_value = {
"error":{
"code": "00001",
"message": "error message for foo reason"
}
}
As it was originally written, getSomething would have had to have called response() directly.
error_json = error.response()
Here is a more generic example:
>>> from unittest.mock import Mock
>>> mock_response = Mock()
>>> mock_response.return_value = {"hello": "world"}
>>> mock_response() # call response() directly
{'hello': 'world'}
>>> mock_response.json() # no return_value is assigned to the json() call
<Mock name='mock.json()' id='4407756744'>
>>> mock_response.json.return_value = {"hello": "universe"} # assign a json() return_value
>>> mock_response.json()
{'hello': 'universe'}
If you want, you can even do this while instantiating Mock:
I'm trying to raise an exception in FastAPI when an object with a specific key already exists (e.g. RethinkDb return "Duplicated key" error). Probably something wrong with my method logic, but can't get what exactly.
#router.post("/brands", response_model=Brand, status_code=status.HTTP_201_CREATED)
def add_brand(brand: Brand):
with r.connect('localhost', 28015, 'expressparts').repl() as conn:
try:
result = r.table("brands").insert({
"id": brand.id,
"name": brand.name}).run(conn)
if result['errors'] > 0:
error = result['first_error'].split(":")[0]
raise HTTPException(
status_code=400, detail=f"Error raised: {error}")
else:
return brand
except Exception as err:
print(err)
You have a try-catch and it captures all the errors that occured. You are just capturing your own Exception which is isn't actually been raised yet.
#router.post("/brands", response_model=Brand, status_code=status.HTTP_201_CREATED)
def add_brand(brand: Brand):
with r.connect('localhost', 28015, 'expressparts').repl() as conn:
result = r.table("brands").insert({
"id": brand.id,
"name": brand.name}).run(conn)
if result['errors'] > 0:
error = result['first_error'].split(":")[0]
raise HTTPException(
status_code=400, detail=f"Error raised: {error}")
else:
return brand
This should be working fine.
Hi I am experiencing weird behavior from SimpleHttpOperator.
I have extended this operator like this:
class EPOHttpOperator(SimpleHttpOperator):
"""
Operator for retrieving data from EPO API, performs token validity check,
gets a new one, if old one close to not valid.
"""
#apply_defaults
def __init__(self, entity_code, *args, **kwargs):
super().__init__(*args, **kwargs)
self.entity_code = entity_code
self.endpoint = self.endpoint + self.entity_code
def execute(self, context):
try:
token_data = json.loads(Variable.get(key="access_token_data", deserialize_json=False))
if (datetime.now() - datetime.strptime(token_data["created_at"],
'%Y-%m-%d %H:%M:%S.%f')).seconds >= 19 * 60:
Variable.set(value=json.dumps(get_EPO_access_token(), default=str), key="access_token_data")
self.headers = {
"Authorization": f"Bearer {token_data['token']}",
"Accept": "application/json"
}
super(EPOHttpOperator, self).execute(context)
except HTTPError as http_err:
logging.error(f'HTTP error occurred during getting EPO data: {http_err}')
raise http_err
except Exception as e:
logging.error(e)
raise e
And I have written a simple unit test:
def test_get_EPO_data(requests_mock):
requests_mock.get('http://ops.epo.org/rest-services/published-data/publication/epodoc/EP1522668',
text='{"text": "test"}')
requests_mock.post('https://ops.epo.org/3.2/auth/accesstoken',
text='{"access_token":"test", "status": "we just testing"}')
dag = DAG(dag_id='test_data', start_date=datetime.now())
task = EPOHttpOperator(
xcom_push=True,
do_xcom_push=True,
http_conn_id='http_EPO',
endpoint='published-data/publication/epodoc/',
entity_code='EP1522668',
method='GET',
task_id='get_data_task',
dag=dag,
)
ti = TaskInstance(task=task, execution_date=datetime.now(), )
task.execute(ti.get_template_context())
assert ti.xcom_pull(task_ids='get_data_task') == {"text": "test"}
Test doesn't pass though, the XCOM value from HttpHook is never pushed as an XCOM, I have checked that code responsible for the push logic in the hook class gets called:
....
if self.response_check:
if not self.response_check(response):
raise AirflowException("Response check returned False.")
if self.xcom_push_flag:
return response.text
What did I do wrong? Is this a bug?
So I actually managed to make it work by setting an xcom value to the result of super(EPOHttpOperator, self).execute(context).
def execute(self, context):
try:
.
.
.
self.headers = {
"Authorization": f"Bearer {token_data['token']}",
"Accept": "application/json"
}
super(EPOHttpOperator, self).execute(context) -> Variable.set(value=super(EPOHttpOperator, self).execute(context),key='foo')
Documentation is kind of misleading on this one; or am I doing something wrong after all?