We Keep Coding

Can not deserialize instance of io.vavr.collection.Seq out of VALUE_STRING

I was trying to build a python request module passing API tokens, however I am running into below issue every time. Sharing my entire code :
import requests
import json
def bitbucketFunction():
variables = {"test":"1234"}
callBitBucketAPI(variables)
def callBitBucketAPI(variables):
try:
headers = {
'Content-Type': 'application/json'
}
url = "https:xyz/pipelines/"
data = {
"target": {
"type": "pipeline_ref_target",
"ref_type": "branch",
"ref_name": "master",
"selector": {
"type": "custom",
"pattern": "create-aws-account"
}
},"variables":json.dumps(variables, indent=2)}
response = requests.request("POST",url, auth=('token-key', 'token-value'), data=json.dumps(data),headers=headers)
print(response.text)
except Exception as e:
print("Exception occured in callBitBucketAPI method: ", e)
bitbucketFunction()
Error:
{"error": {"message": "An invalid field was found in the JSON payload.", "fields": ["variables"], "detail": "Can not deserialize instance of io.vavr.collection.Seq out of VALUE_STRING token\n at [Source: (org.glassfish.jersey.message.internal.ReaderInterceptorExecutor$UnCloseableInputStream); line: 1, column: 167] (through reference chain: com.atlassian.pipelines.rest.model.v1.pipeline.ImmutablePipelineModel$Builder[\"variables\"])", "data": {"key": "rest-service.request.invalid-json"}}}

How to mock API request used inside function in a Django test?

I've got a little utility function built like this to grab data from another applications API:
# app/utils.py
import json
import requests
from django.conf import settings
def get_future_assignments(user_id):
"""gets a users future assignments list from the API
Arguments:
user_id {int} -- user_id for a User
"""
headers = {
"User-Agent": "Mozilla/5.0",
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
"X-Requested-With": "XMLHttpRequest",
}
api_app = settings.ASSIGNMENTS_API_ROOT_URL # http://project.org/appname/
api_model = "futureassignments/"
api_query = "?user_id=" + str(user_id)
json_response = requests.get(
api_app + api_model + api_query, headers=headers, verify=False
)
return json.loads(json_response.content)
It basically builds the API call and returns the response data - I'd like to test this.
# tests/test_utils.py
import mock
from unittest.mock import patch, Mock
from django.test import TestCase
from app.utils import get_future_assignments
class UtilsTest(TestCase):
def setUp(self):
self.futureassignments = [
{
"id": 342,
"user_id": 18888,
"job": 361,
"location": "1234",
"building": "Building One",
"confirmed_at": None,
"returning": None,
"signature": None,
},
{
"id": 342,
"user_id": 18888,
"job": 361,
"location": "1235",
"building": "Building Two",
"confirmed_at": None,
"returning": None,
"signature": None,
},
]
#patch("app.utils.get_future_assignments")
def test_get_future_assignments_with_multi_assignments(self, mock_gfa):
"""
Test for getting future assignments for a user with mocked API
"""
mock_gfa.return_value = Mock()
# set the json response to what we're expecting
mock_gfa.return_value.json.return_value = self.futureassignments
assignments = get_future_assignments(18888)
self.assertEqual(len(assignments), 2)
It keeps giving me an error that it can't reach the API to get a response (which is expected at the moment - since I'm running it locally and it cannot hit the API)
I'm new to using Mock - so maybe I'm way, way off base here.
Any ideas?

Like you, I am also new to using mock. I believe it's intended to work like this:
#patch("requests.get")
def test_get_future_assignments_with_multi_assignments(self, mock_requests_get):
"""
Test for getting future assignments for a user with mocked API
"""
mock_requests_get.return_value = json.dumps(self.futureassignments)
assignments = get_future_assignments(18888)
self.assertEqual(len(assignments), 2)
Please correct me if I'm wrong!

Slimming JSON messages down without deriving from `list`

Consider a server generating JSON messages for a to-do list.
import json
class Action(object):
def __init__(self, what, when):
self.what = what
self.when = when
class Actions(object):
def __init__(self):
self.actions = []
def insert(self, action):
self.actions.append({'action': 'insert_todo',
'detail': {'what': action.what,
'when': action.when}})
class Batch(object):
def __init__(self):
self.urgent_actions = Actions()
self.ordinary_actions = Actions()
self.urgent_actions.insert(Action('tidy up', '8am'))
def jdefault(o):
return o.__dict__
def output_json():
batch = Batch()
mystr = json.dumps(batch,
default=jdefault,
indent=4)
print(mystr)
output_json()
This works fine, and we get the message:
{
"urgent_actions": {
"actions": [
{
"action": "insert_todo",
"detail": {
"what": "tidy up",
"when": "8am"
}
}
]
},
"ordinary_actions": {
"actions": []
}
}
But repeating actions inside both priorities of actions and in each message is asking for some clean-up.
We can do that by deriving Actions from list:
class Actions(list):
def __init__(self, *args):
list.__init__(self, *args)
def insert(self, action):
self.append({'action': 'insert_todo',
'detail': {'what': action.what,
'when': action.when}})
And we get indeed the slimmer JSON message:
{
"urgent_actions": [
{
"action": "insert_todo",
"detail": {
"what": "8am",
"when": "8am"
}
}
],
"ordinary_actions": []
}
Yet,
deriving
from
list
is
far
from
the best idea.
What other (idiomatic) way would you use to get the slimmer message without deriving from list?
The messages are to be sent through Flask, in case you'd also like to critique the use of json.dumps.

Rather than change the classes, you could delegate defining the structure of the json to the jdefault function.
def jdefault(o):
if isinstance(o, Batch):
return o.__dict__
if isinstance(o, Actions):
return o.actions
raise TypeError("Object of type {} is not JSON serializable".format(type(o)))
Which generates the desired output:
{
"urgent_actions": [
{
"action": "insert_todo",
"detail": {
"what": "tidy up",
"when": "8am"
}
}
],
"ordinary_actions": []
}
This way you separate the concerns of your objects' structure and serialisation.

Flask-restplus returning marshal model instead of the data

So I'm pretty new to implementing flask-restplus and I have encountered this road block.
I have read the restplus docs over and over again and followed several exampled. But the behavior that I'm facing is very much different from what is supposed to be.
So I have a model that is supposed to be a list of objects of another model (returned from the function drone_model()).
drones_list = api.model('drones_list', {
'items': fields.List(fields.Nested(drone_model())),
'message':fields.String(''),
'code': fields.Integer('')
})
Everything works fine, no errors. But when I try the API (http://127.0.0.1:5000/datamine/v2/drones), as a response I get the Marshalling model back instead of the data itself. If I print the data, it gets printed, but for some reason in the web, the restplus model is returned.
Below I have the code that I had written. If I take the marshal_with decorator off, then the data is returned just fine.
#api.route('/')
class DronesList(Resource):
#api.marshal_with(drones_list, envelope='data')
#api.response(200, 'All drones successfully fetched!')
def get(self):
"""
Get all drones!.
"""
from app.utils.common import get_start_end_date_from_request
start_date, end_date = get_start_end_date_from_request(request)
drones = []
for drone in Drone.objects:
drones.append({
'id': str(drone.id),
'serial_id': drone.serial_id,
'maintenances': [],
'status': get_dynamic_status(drone, start_date, end_date),
'picture_url': drone.asset.picture_url,
'manufacturer': drone.asset.manufacturer,
'model_name': drone.asset.model_name,
'drone_type': drone.asset.drone_type,
'payload_type': drone.asset.payload_type,
'asset_url': drone.get_url(drone.id)
})
success = ClientSuccessFunctionClass('All drones successfully fetched!', 200, drones)
return (success.to_dict())
These are the outputs on the browser:
1. Without the marshal decorator:
{
"data": {
"items": [
{
"id": "5aeafcb93a33683f73827e91",
"serial_id": "Drone 1",
"maintenances": [],
"status": "Decommissioned",
"picture_url": "some img url",
"manufacturer": "DJI",
"model_name": "Phantom 4 Pro",
"drone_type": "Quadcopter",
"payload_type": "RGB Camera",
"asset_url": "http://127.0.0.1:5000/datamine/v1/drones/5aeafcb93a33683f73827e91"
},
{
"id": "5aeaff374f85747f90df2714",
"serial_id": "Drone 2",
"maintenances": [],
"status": "Available",
"picture_url": "sime url",
"manufacturer": "DJI",
"model_name": "Phantom 4",
"drone_type": "Quadcopter",
"payload_type": "RGB Camera",
"asset_url": "http://127.0.0.1:5000/datamine/v1/drones/5aeaff374f85747f90df2714"
}
],
"message": "All drones successfully fetched!",
"code":200
}
}
2. With the marshal decorator:
{
"data": {
"items": [
{
"id": "Id of Drone",
"serial_id": "Name of Drone",
"status": "Status of Drone",
"maintenances": null,
"picture_url": "Picture URL",
"manufacturer": "Manufacturer of Drone",
"model_name": "Model name of Drone",
"drone_type": "Type of Drone",
"payload_type": "Payload type of Drone",
"asset_url": "Asset URL of Drone"
}
],
"message": "",
"code": ""
}
}
It would be really helpful if someone could tell me what I'm doing wrong as I need to recive the output as the one shown in snippet of the output without the decorator.
Thank you.

Here is a diagram showing invocation order from top to bottom to help make sense of what is happening:
get()
→ api.response(200, 'All drones successfully fetched!') # documents the response
→ api.marshal_with(drones_list, envelope='data')` # returns marshalled dict
The result from invoking get is passed to the api.response decorator function whose result is passed on to api.marshal_with decorator function.
Looking at the shape of the dictionary returned from invoking get()
{
data {
items [
{
id,
serial_id,
maintenances,
status,
picture_url,
manufacturer,
model_name,
drone_type,
payload_type,
asset_url
}
],
message,
code
}
}
The message and code in the response are nested inside of the data.
You need to model the data appropriately, to be able to marshal it. This can be done by passing an argument for what field to look up in the marshal dictionary.
drones_list = api.model('drones_list', {
'items': fields.List(fields.Nested(drone_model()), attribute='data.items'),
'message':fields.String(attribute='data.message'),
'code': fields.Integer(attribute='data.code')
})
As you can see, it's pretty redundant applying the api.marshal_with decorator function on the view given that it's only unnests then nests the result in data field.

Python: Mocking DateTime Issues

Here is my Python method:
for time in (timedelta(hours=3), timedelta(minutes=30)):
delay = (datetime.now() + time).strftime('%Y-%m-%dT%H:%M:%S')
payload = json.dumps({
"channels": [
"accountID-{acct_id}".format(acct_id=account_id)
],
"push_time": delay,
"data": {
"alert": "Test Push",
"m": "12345"
},
})
try:
requests.post(
"https://api.parse.com/1/push",
data=payload,
headers={
"X-Parse-Application-Id": settings.PARSE_APPLICATION_ID,
"X-Parse-REST-API-Key": settings.PARSE_REST_API_KEY,
"Content-Type": "application/json"
}
)
except requests.exceptions.RequestException as e:
logging.getLogger().setLevel(logging.ERROR)
Here is my test:
#patch("requests.post")
def test_send_push_notifications_to_parse(self, post_mock):
post_mock.return_value = {"status": 200}
mailing = Mock()
mailing.name = "Foo Name"
mailing.account_id = 12345
mailing.mailing_id = 45
payload = json.dumps({
"channels": [
"accountID-12345"
],
"push_time": "2014-03-04T15:00:00",
"data": {
"alert": "Test Push",
"m": "12345"
},
})
send_push_notification_to_parse(mailing.account_id, mailing.mailing_id, mailing.name)
post_mock.assert_called_with(
"https://api.parse.com/1/push",
data=payload,
headers={
"X-Parse-Application-Id": settings.PARSE_APPLICATION_ID,
"X-Parse-REST-API-Key": settings.PARSE_REST_API_KEY,
"Content-Type": "application/json"
}
)
The test fails because the POST request is inside of a loop where the datetimeobject changes. How can I patch the datetime object to make my test pass?

Just mock datetime in your module:
#patch("your.module.datetime")
#patch("requests.post")
def test_send_push_notifications_to_parse(self, post_mock, dt_mock):
# set datetime.now() to a fixed value
dt_mock.now.return_value = datetime.datetime(2013, 2, 1, 10, 9, 8)
You bound datetime in your module with an import, and the above #patch decorator will replace that object with a mock.
If you need to test against multiple values, you can set the dt_mock.now.side_effect attribute instead; a list will cause the mock to return values one by one from that list on sequential calls, a method lets you produce a new value each time datetime.now() is called as needed.

Anyone stumbling on this old question like me can use freezegun that Lets your Python tests travel through time(like Interstellar:))