We Keep Coding

Mock a boto3 (botocore) ClientError for a dynamodb Table update_item call

I am writing a unit test for a python function that does a update_item in dynamodb via the table resource. The normal unit test works but raising a ClientError is something I cannot ge to work..
The tests get executed but I alwys:
E Failed: DID NOT RAISE <class 'botocore.exceptions.ClientError'>
What I am trying to do is simulate the case when I would get a Internal Server Error of AWS or a RequestLimitExceeded error.
Thanks for pointing me in the correct direction!
My functional code:
def _updateCardConfiguration(identifier, cardType, layout):
try:
table.update_item(
Key={
'identifier': identifier,
'cardType': cardType
},
UpdateExpression="SET layout = :layout",
ExpressionAttributeValues={
':layout': layout
}
)
except ClientError as e:
logger.fatal(
f"The update of the cardconfiguration for the niche: {identifier} has failed with the following message: {e.response['Error']['Message']}")
raise
return True
def lambda_handler(event, context):
# Check the cardtype
cardType = 'SearchresultCard' if (
event['detail']['action'] == 'updateSearchresultCard') else 'DetailCard'
# Update the card configuration
_updateCardConfiguration(
event['detail']['identifier'], cardType, event['detail']["layout"])
return True
My test code:
#pytest.fixture(scope='function')
def cardConfigurationsTable(aws_credentials):
with mock_dynamodb2():
#client = boto3.client('dynamodb', region_name='eu-west-1')
dynamodb = boto3.resource('dynamodb')
dynamodb.create_table(
TableName='CardConfigurations',
KeySchema=[
{
'AttributeName': 'identifier',
'KeyType': 'HASH'
},
{
'AttributeName': 'cardType',
'KeyType': 'RANGE'
}
],
AttributeDefinitions=[
{
'AttributeName': 'identifier',
'AttributeType': 'S'
},
{
'AttributeName': 'cardType',
'AttributeType': 'S'
},
],
StreamSpecification={
'StreamEnabled': True,
'StreamViewType': 'NEW_AND_OLD_IMAGES'
},
BillingMode='PAY_PER_REQUEST',
SSESpecification={
'Enabled': True
},
GlobalSecondaryIndexes=[
{
'IndexName': 'byIdentifier',
'KeySchema': [
{
'AttributeName': 'identifier',
'KeyType': 'HASH'
},
{
'AttributeName': 'cardType',
'KeyType': 'RANGE'
}
],
'Projection': {
'ProjectionType': 'ALL'
}
}
])
yield dynamodb.Table('CardConfigurations')
def test_updateItem_raises_clientExecption(cardConfigurationsTable, env_vars):
""" Unit test for testing the lambda that updates the cardconfiguration
Scenario: No detail card is configured for the niche
:param fixture cardConfigurationsTable: The fixture that mocks the dynamodb CardConfigurations table
:param fixture env_vars: The fixture with the env vars
"""
# Prepare the event
with open('tests/unit/event-flows/dataconfiguration-state-transition/events/updateSearchresultCard.json') as json_file:
event = json.load(json_file)
stubber = Stubber(cardConfigurationsTable.meta.client)
stubber.add_client_error(
"update_item",
service_error_code="InternalServerError",
service_message="Internal Server Error",
http_status_code=500,
)
stubber.activate()
# Import the lambda handler
lambdaFunction = importlib.import_module(
"event-flows.dataconfiguration-state-transition.updateCardconfigurationInDb.handler")
with pytest.raises(ClientError) as e_info:
# Execute the handler
response = lambdaFunction.lambda_handler(event, {})

I would start by creating a default table that your _updateCardConfiguration function can use.
def get_table():
table = dynamodb.Table("CardConfigurations")
return table
default_table = get_table()
You can then pass it as a default argument to the function that needs it. This makes it easier to test the function. To do so, modify your _updateCardConfiguration function header to look like this :
def _updateCardConfiguration(identifier, cardType, layout, table=default_table):
Doing this gives us the ability to replace the default table with a mocked table for testing. The complete modified function looks like this:
def _updateCardConfiguration(identifier, cardType, layout, table=default_table):
try:
table.update_item(
Key={"identifier": identifier, "cardType": cardType},
UpdateExpression="SET layout = :layout",
ExpressionAttributeValues={":layout": layout},
)
except ClientError as e:
logger.fatal(
f"The update of the cardconfiguration for the niche: {identifier} has failed with the following message: {e.response['Error']['Message']}"
)
raise
return True
As you can see, all that changes is the header.
Then in your test_updateItem_raises_clientExecption test function file, I would change the lines
with pytest.raises(ClientError) as e_info:
# Execute the handler
response = lambdaFunction.lambda_handler(event, {})
to:
with pytest.raises(ClientError, match="InternalServerError"):
# Execute the _updateCardConfiguration function
lambdaFunction._updateCardConfiguration(
None, None, None, cardConfigurationsTable
)
By passing in your cardConfigurationsTable fixture, your function wiil operate against it and you should get the behaviour you expect.

How to specify device fingerprint id in Cybersource REST API?

I'm trying to integrate CyberSource's REST API into a Django (Python) application. I'm following this GitHub example example.
It works like a charm but it is not clear to me from the example or from the documentation how to specify the device's fingerprint ID.
Here's a snippet of the request I'm sending in case it comes useful (note: this is just a method that lives inside a POPO):
def authorize_payment(self, card_token: str, total_amount: Money, customer: CustomerInformation = None,
merchant: MerchantInformation = None):
try:
request = {
'payment_information': {
# NOTE: REQUIRED.
'card': None,
'tokenized_card': None,
'customer': {
'customer_id': card_token,
},
},
'order_information': {
'amount_details': {
'total_amount': str(total_amount.amount),
'currency': str(total_amount.currency),
},
},
}
if customer:
request['order_information'].update({
'bill_to': {
'first_name': customer.first_name,
'last_name': customer.last_name,
'company': customer.company,
'address1': customer.address1,
'address2': customer.address2,
'locality': customer.locality,
'country': customer.country,
'email': customer.email,
'phone_number': customer.phone_number,
'administrative_area': customer.administrative_area,
'postalCode': customer.zip_code,
}
})
serialized_request = json.dumps(request)
data, status, body = self._payment_api_client.create_payment(create_payment_request=serialized_request)
return data.id
except Exception as e:
raise AuthorizePaymentError from e

mediaItems.search not working with albumId

When I run the following code I get this error.
{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name "album_id": Proto field is not repeating, cannot start list.', 'status': 'INVALID_ARGUMENT', 'details': [{'#type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'description': 'Invalid JSON payload received. Unknown name "album_id": Proto field is not repeating, cannot start list.'}]}]}}
If I remove the "albumId": ["albumid code"] it works fine and returns
10 new items, total 10
def _actually_list_media_items(session):
ret = []
params = {
'fields': 'mediaItems(id,baseUrl,filename,mimeType,productUrl),nextPageToken',
}
search_json = {
"pageSize": 10,
"albumId": ["<albumid code>"],
"filters": {
"includeArchivedMedia": False,
"contentFilter": {
"excludedContentCategories": [
"DOCUMENTS",
"RECEIPTS",
"SCREENSHOTS",
"UTILITY",
"WHITEBOARDS",
]
},
"mediaTypeFilter": {
"mediaTypes": [
"PHOTO",
],
},
},
}
tmp = 0
while tmp < 1:
rsp = session.post(
'https://photoslibrary.googleapis.com/v1/mediaItems:search',
params=params,
json=search_json,
).json()
if 'error' in rsp:
print(rsp)
cur = [m for m in rsp.get('mediaItems', [])]
ret += cur
print(f'{len(cur)} new items, total {len(ret)}')
pageToken = rsp.get('nextPageToken')
if pageToken is None:
break
params['pageToken'] = pageToken
tmp = tmp + 1
return ret

The comment about albumId and filters being exclusive is correct, so you need to pick one or the other. However, assuming you want to use the albumId by itself, you need to remove the square brackets around your albumid code, here's a clip from my code:
searchbody = {
"albumId": album_id,
"pageSize": 10
}
print(searchbody)
mediaresults = gAPIservice.mediaItems().search(body=searchbody).execute()
mediaitems = mediaresults.get('mediaItems', [])
for item in mediaitems:
print(u'{0} ({1})'.format(item['filename'], item['id']))

Edit:
Apparently you can't use albumId and filters together: source
filters: object(Filters)
Filters to apply to the request. Can't be set in conjunction with an albumId.
Aside from that, albumId is a supposed to be a string not an array: source
"albumId": "<albumid code>",

Google DLP: "ValueError: Protocol message Value has no "stringValue" field."

I have a method where I build a table for multiple items for Google's DLP inspect API which can take either a ContentItem, or a table of values
Here is how the request is constructed:
def redact_text(text_list):
dlp = google.cloud.dlp.DlpServiceClient()
project = 'my-project'
parent = dlp.project_path(project)
items = build_item_table(text_list)
info_types = [{'name': 'EMAIL_ADDRESS'}, {'name': 'PHONE_NUMBER'}]
inspect_config = {
'min_likelihood': "LIKELIHOOD_UNSPECIFIED",
'include_quote': True,
'info_types': info_types
}
response = dlp.inspect_content(parent, inspect_config, items)
return response
def build_item_table(text_list):
rows = []
for item in text_list:
row = {"values": [{"stringValue": item}]}
rows.append(row)
table = {"table": {"headers": [{"name": "something"}], "rows": rows}}
return table
When I run this I get back the error ValueError: Protocol message Value has no "stringValue" field. Even though the this example and the docs say otherwise.
Is there something off in how I build the request?
Edit: Here's the output from build_item_table
{
'table':
{
'headers':
[
{'name': 'value'}
],
'rows':
[
{
'values':
[
{
'stringValue': 'My name is Jenny and my number is (555) 867-5309, you can also email me at anemail#gmail.com, another email you can reach me at is email#email.com. '
}
]
},
{
'values':
[
{
'stringValue': 'Jimbob Doe (555) 111-1233, that one place down the road some_email#yahoo.com'
}
]
}
]
}
}

Try string_value .... python uses the field names, not the type name.

Flask-restful: marshal complex object to json

I have a question regarding flask restful extension. I'm just started to use it and faced one problem. I have flask-sqlalchemy entities that are connected many-to-one relation and I want that restful endpoint return parent entity with all its children in json using marshaller. In my case Set contains many parameters. I looked at flask-restful docs but there wasn't any explanation how to solve this case.
Seems like I'm missing something obvious but cannot figure out any solution.
Here is my code:
# entities
class Set(db.Model):
id = db.Column("id", db.Integer, db.Sequence("set_id_seq"), primary_key=True)
title = db.Column("title", db.String(256))
parameters = db.relationship("Parameters", backref="set", cascade="all")
class Parameters(db.Model):
id = db.Column("id", db.Integer, db.Sequence("parameter_id_seq"), primary_key=True)
flag = db.Column("flag", db.String(256))
value = db.Column("value", db.String(256))
set_id = db.Column("set_id", db.Integer, db.ForeignKey("set.id"))
# marshallers
from flask.ext.restful import fields
parameter_marshaller = {
"flag": fields.String,
"value": fields.String
}
set_marshaller = {
'id': fields.String,
'title': fields.String,
'parameters': fields.List(fields.Nested(parameter_marshaller))
}
# endpoint
class SetApi(Resource):
#marshal_with(marshallers.set_marshaller)
def get(self, set_id):
entity = Set.query.get(set_id)
return entity
restful_api = Api(app)
restful_api.add_resource(SetApi, "/api/set/<int:set_id>")
Now when i call /api/set/1 I get server error:
TypeError: 'Set' object is unsubscriptable
So I need a way to correctly define set_marshaller that endpoint return this json:
{
"id": : "1",
"title": "any-title",
"parameters": [
{"flag": "any-flag", "value": "any-value" },
{"flag": "any-flag", "value": "any-value" },
.....
]
}
I appreciate any help.

I found solution to that problem myself.
After playing around with flask-restful i find out that i made few mistakes:
Firstly set_marshaller should look like this:
set_marshaller = {
'id': fields.String,
'title': fields.String,
'parameters': fields.Nested(parameter_marshaller)
}
Restless marshaller can handle case if parameter is list and marshals to json list.
Another problem was that in API Set parameters has lazy loading, so when i try to marshall Set i got KeyError: 'parameters', so I need explicitly load parameters like this:
class SetApi(Resource):
#marshal_with(marshallers.set_marshaller)
def get(self, set_id):
entity = Set.query.get(set_id)
entity.parameters # loads parameters from db
return entity
Or another option is to change model relationship:
parameters = db.relationship("Parameters", backref="set", cascade="all" lazy="joined")

This is an addition to Zygimantas's answer:
I'm using Flask-RESTful and this is a solution to the loading of the nested properties.
You can pass a callable to the marshal decorator:
class OrgsController(Resource):
#marshal_with(Organization.__json__())
def get(self):
return g.user.member.orgs
Then update the models to return the resource fields for its own entity. Nested entities will thus return the resource fields for its entity relatively.
class Organization(db.Model):
id = db.Column(db.Integer, primary_key=True)
...
#staticmethod
def __json__(group=None):
_json = {
'id': fields.String,
'login': fields.String,
'description': fields.String,
'avatar_url': fields.String,
'paid': fields.Boolean,
}
if group == 'flat':
return _json
from app.models import Repository
_json['repos'] = fields.Nested(Repository.__json__('flat'))
return _json
class Repository(db.Model):
id = db.Column(db.Integer, primary_key=True)
owner_id = db.Column(db.Integer, db.ForeignKey('organization.id'))
owner = db.relationship('Organization', lazy='select', backref=db.backref('repos', lazy='select'), foreign_keys=[owner_id])
...
#staticmethod
def __json__(group=None):
_json = {
'id': fields.String,
'name': fields.String,
'updated_at': fields.DateTime(dt_format='iso8601'),
}
if group == 'flat':
return _json
from app.models import Organization
_json['owner'] = fields.Nested(Organization.__json__('flat'))
return _json
This gives the representation I'm looking for, and honoring the lazy loading:
[
{
"avatar_url": "https://avatars.githubusercontent.com/u/18945?v=3",
"description": "lorem ipsum.",
"id": "1805",
"login": "foobar",
"paid": false,
"repos":
[
{
"id": "9813",
"name": "barbaz",
"updated_at": "2014-01-23T13:51:30"
},
{
"id": "12860",
"name": "bazbar",
"updated_at": "2015-04-17T11:06:36"
}
]
}
]
I like
1) how this approach allows me to define my resource fields per entity and it is available to all my resource routes across the app.
2) how the group argument allows me to customise the representation however I desire. I only have 'flat' here, but any logic can be written and passed down to deeper nested objects.
3) entities are only loaded as necessary.