I try to test a file upload like this:
#deconstructible
class FileGenerator:
#staticmethod
def generate_text_file(file_ending='txt'):
file_content = b'some test string'
file = io.BytesIO(file_content)
file.name = f'test.{file_ending}'
file.seek(0)
return file
def test_this(self, api_client, login_as):
user = login_as('quality-controller')
url = reverse('test-list')
organization = Organization(name="test")
organization.save()
data = {
"organization": organization.id,
"import_file": FileGenerator.generate_text_file('txt'),
"user": {
"id": user.id,
"username": user.username,
}
}
response = api_client.post(url, data, format='json')
But I receive the following error message:
b'{"import_file": ["The submitted data was not a file. Check the
encoding type on the form."]}'
I also tried to use: format='multipart' but then I receive the following error:
AssertionError: Test data contained a dictionary value for key 'user',
but multipart uploads do not support nested data. You may want to
consider using format='json' in this test case.
How can I solve this?
This is how I deal with this issue:
Simplest: flatten the form
Suck it up and just remove the issue by making your serializer to use user_id and user_username and fix it up on the server side in the serializer's validate(self, attrs) method. A bit ugly/hacky but it works just fine and can be documented.
def validate(self, attrs):
attrs["user"] = {
"id": attrs.pop("user_id"),
"name": attrs.pop("user_username")
}
return attrs
Nicest if you dont mind the size: B64 Fields
You can base64 encode the file field and pass it in the json. Then to decode it on the server side you would write (or search for) a simple Base64FileField() for DRF.
class UploadedBase64ImageSerializer(serializers.Serializer):
file = Base64ImageField(required=False)
created = serializers.DateTimeField()
Alternative - Flatten the form data
You can't pass nested data, but you can flatten the nested dicts and pass that to a DRF service. Serializers actually can understand nested data if the field names are correct.
I don't know if this field name format is standardized, but this is what worked for me after experimentation. I only use it for service->service communication TO drf, so you would have to clone it into JS, but you can use the python in unit tests. Let me know if it works for you.
def flatten_dict_for_formdata(input_dict, array_separator="[{i}]"):
"""
Recursively flattens nested dict()s into a single level suitable
for passing to a library that makes multipart/form-data posts.
"""
def __flatten(value, prefix, result_dict, previous=None):
if isinstance(value, dict):
# If we just processed a dict, then separate with a "."
# Don't do this if it is an object inside an array.
# In that case the [:id] _is_ the separator, adding
# a "." like list[1].name will break but list[x]name
# is correct (at least for DRF/django decoding)
if previous == "dict":
prefix += "."
for key, v in value.items():
__flatten(
value=v,
prefix=prefix + key,
result_dict=result_dict,
previous="dict"
)
elif isinstance(value, list) or isinstance(value, tuple):
for i, v in enumerate(value):
__flatten(
value=v,
prefix=prefix + array_separator.format(i=i), # e.g. name[1]
result_dict=result_dict,
previous="array"
)
else:
result_dict[prefix] = value
# return her to simplify the caller's life. ignored during recursion
return result_dict
return __flatten(input_dict, '', OrderedDict(), None)
# flatten_dict_for_formdata({...}):
{ # output field name
"file": SimpleUploadFile(...), # file
"user": {
"id": 1, # user.id
"name": "foghorn", # user.name
"jobs": [
"driver", # user.jobs[0]
"captain", # user.jobs[1]
"pilot" # user.jobs[1]
]
},
"objects": [
{
"type": "shoe", # objects[0]type
"size": "44" # objects[0]size
},
]
}
Related
I was new to python, my requirement is to check whether the given key exists on the json or not. Json will not be same all the time. So, am looking for the generic function to check whether the key exists. It works for simple json , but returns nothing when the json itself has another json with an jsonarray inside as shown below:
{
"id": "1888741f-173a-4366-9fa0-a156d8734972",
"type": "events",
"version": "1.0.0",
"count": 3,
"payload": {
"cmevents": [
{
"exit_code": "0dbc2745-a964-4ce3-b7a0-bd5295afc620",
"sourceEventType": "test01",
"sourceType": "test",
product:{
"productCode":"101"
}
},
{
"exit_code": "1dbc2745-a964-4ce3-b7a0-bd5295afc620",
"sourceEventType": "test02",
"sourceType": "test",
product:{
"productCode":"102"
}
},
{
"exit_code": "2dbc2745-a964-4ce3-b7a0-bd5295afc620",
"sourceEventType": "test03",
"sourceType": "test",
product:{
"productCode":"103"
}
}
]
}
}
from the above json , i wants to check the key sourceEventType exists in all items of cmevents list.
Following is the function I have used
def checkElementsExist(element, JSON, path, whole_path):
if element in JSON:
path = path + element + ' = ' + JSON[element].encode('utf-8')
whole_path.append(path)
//
for key in JSON:
if isinstance(JSON[key], dict):
finds(element, JSON[key], path + key + '.', whole_path)
To call the function:
whole_path = []
finds('sourceEventType', json, '', whole_path)
Can anyone please help me with the right solution
A recursive function is probably an easier approach here.
Parse the json using json.loads(text)
Search the tree
import json
text = json.loads(".....")
def search_for_key(key, content) -> bool:
# Search each item of the list
# if found return true
if isinstance(content, list):
for elem in content:
if search_for_key(key, elem):
return True
# Search each key of the dictionary for match
# If the key isn't a match try searching the value
# of the key in the dictionary
elif isinstance(content, dict):
for key_in_json in content:
if key_in_json == key:
return True
if search_for_key(key, content[key_in_json]):
return True
# If here it's a number or string. There is no where else to go
# return false
else:
return False
print(search_for_key("some_key?", text))
It's definitely possible to make your path approach work BUT you're doing need to keep track of what paths you haven't explored yet using a stack/queue (and you get this for free with a recursive function).
I'm new to Python so sorry in advance if the answer to this is a basic concept.
I'm struggling with trying to identify when a parameter is None or is not received.
I wrote a quick example below to illustrate my doubt.
Considering I have the following Enum that I'll use as a datatype:
class Status(Enum):
ACTIVE = "1"
PENDING = "2"
DELETED = "3"
I want to have a single function that can do 3 scenarios:
Build a dict with a field with a value:
{"name": "John", "status": "1"}
Build a dict with a field with null value:
{"name": "John", "status": None}
Build a dict without a field:
{"name": "John"}
For that, I created 3 tests:
def test_send_valid_status():
request_body = build_request(name="John", status=Status.ACTIVE)
assert body == {"name": "John", "status": "1"}
def test_send_null_status():
request_body = build_request(name="John", status=None)
assert body == {"name": "John", "status": None}
def test_dont_send_status():
request_body = build_request(name="John")
assert body == {"name": "John"}
How can I write build_request() in order to pass the above scenarios? Is important to me that the person calling build_request() knows there's a field status that can be sent and what datatype should be.
My first attempt was to make status optional:
def build_request(name: str, status: bool = None):
request_body = {
"name": name,
}
if status is not None:
request_body["status"] = status.value
return request_body
Of course this fails on test_dont_send_status() because status is assigned a None if I don't send the parameter.
My second attempt was to use kwargs and tell the user in the docs what to send:
def build_request(name: str, **kwargs):
"""
:param name: str
:param status: Status = None
:return: dict
"""
request_body = {
"name": name,
}
for arg, value in kwargs.items():
request_body[arg] = value.value
return request_body
The problem here is that pycharm complains param status is not an argument. Also, it doesn't seem to be a robust solution.
How can I handle that difference between the parameter being None and the parameter not being sent?
In general, you cannot differentiate. That's why using **kwargs is the right way.
Pycharm complaints because status indeed is not an argument of this function, it can take anything. Co you must do the type/value check manually (you cannot enforce the enum).
Another option is to use different default value than None, some constant would work. And then compare it to this constant.
NOT_PRESENT = []
def build_request(name: str, status: Union[bool, list] = NOT_PRESENT):
request_body = {
"name": name,
}
if status is not NOT_PRESENT:
request_body["status"] = status.value
return request_body
You can use the dictionary's get() method, which can take a value for the cases when the value is not found in the dict, the default is None here too but you can change it to something that makes sense and signals that the value was not found in the dict. You use it like this:
result = mydict.get(key_of_val_to_be_found, value_if_not_found)
You can put a default value that you won't use.
For example:
def build_request(name: str, status="Empty"):
request_body = { "name": name }
if status == "Empty":
pass
elif status is None:
request_body["status"] = None
elif isinstance(status, bool):
request_body["status"] = int(status)
else:
# manage invalid status here
print("invalid status")
return request_body
Spoiler alert: I posted my solution as an answer to this question
I am using flastk-resptlus to create an API. I have to provide the data in a specific structure, which I have problems to get, see an example below:
What I need to get is this structure:
{
"metadata": {
"files": []
},
"result" : {
"data": [
{
"user_id": 1,
"user_name": "user_1",
"user_role": "editor"
},
{
"user_id": 2
"user_name": "user_2",
"user_role": "editor"
},
{
"user_id": 3,
"user_name": "user_3",
"user_role": "curator"
}
]
}
}
But the problem comes that I cannot manage to get the structure of "result" : { "data": []} without making "data" a model itself.
What I tried to do so far (and did not work)
# define metadata model
metadata_model = api.model('MetadataModel', {
"files": fields.List(fields.String(required=False, description='')),
}
# define user model
user_model = api.model('UserModel', {
"user_id": fields.Integer(required=True, description=''),
"user_name": fields.String(required=True, description=''),
"user_role": fields.String(required=False, description='')
}
# here is where I have the problems
user_list_response = api.model('ListUserResponse', {
'metadata': fields.Nested(metadata_model),
'result' : {"data" : fields.List(fields.Nested(user_model))}
})
Complains that cannot get the "schema" from "data" (because is not a defined model), but I don't want to be a new api model, just want to append a key called "data". Any suggestions?
This I tried and works, but is not what I want (because I miss the "data"):
user_list_response = api.model('ListUserResponse', {
'metadata': fields.Nested(metadata_model),
'result' : fields.List(fields.Nested(user_model))
})
I don't want data to be a model because the common structure of the api is the following:
{
"metadata": {
"files": []
},
"result" : {
"data": [
<list of objects> # here must be listed the single model
]
}
}
Then, <list of objects> can be users, addresses, jobs, whatever.. so I want to make a "general structure" in which then I can just inject the particular models (UserModel, AddressModel, JobModel, etc) without creating a special data model for each one.
A possible approach is to use fields.Raw which returns whatever serializable object you pass. Then, you can define a second function, which creates your result and uses marshal. marshal transforms your data according to a model and accepts an additional parameter called envelope. envelope surrounds your modeled data by a given key and does the trick.
from flask import Flask
from flask_restplus import Api, fields, Resource, marshal
app = Flask(__name__)
api = Api()
api.init_app(app)
metadata_model = api.model("metadata", {
'file': fields.String()
})
user_model = api.model('UserModel', {
"user_id": fields.Integer(required=True, description=''),
"user_name": fields.String(required=True, description=''),
"user_role": fields.String(required=False, description='')
})
response_model = api.model("Result", {
'metadata': fields.List(fields.Nested(metadata_model)),
'result': fields.Raw()
})
#api.route("/test")
class ApiView(Resource):
#api.marshal_with(response_model)
def get(self):
data = {'metadata': {},
'result': self.get_user()}
return data
def get_user(self):
# Access database and get data
user_data = [{'user_id': 1, 'user_name': 'John', 'user_role': 'editor'},
{'user_id': 2, 'user_name': 'Sue', 'user_role': 'curator'}]
# The kwarg envelope does the trick
return marshal(user_data, user_model, envelope='data')
app.run(host='0.0.0.0', debug=True)
My workaround solution that solves all my problems:
I create a new List fields class (it is mainly copied from fields.List), and then I just tune the output format and the schema in order to get the 'data' as key:
class ListData(fields.Raw):
'''
Field for marshalling lists of other fields.
See :ref:`list-field` for more information.
:param cls_or_instance: The field type the list will contain.
This is a modified version of fields.List Class in order to get 'data' as key envelope
'''
def __init__(self, cls_or_instance, **kwargs):
self.min_items = kwargs.pop('min_items', None)
self.max_items = kwargs.pop('max_items', None)
self.unique = kwargs.pop('unique', None)
super(ListData, self).__init__(**kwargs)
error_msg = 'The type of the list elements must be a subclass of fields.Raw'
if isinstance(cls_or_instance, type):
if not issubclass(cls_or_instance, fields.Raw):
raise MarshallingError(error_msg)
self.container = cls_or_instance()
else:
if not isinstance(cls_or_instance, fields.Raw):
raise MarshallingError(error_msg)
self.container = cls_or_instance
def format(self, value):
if isinstance(value, set):
value = list(value)
is_nested = isinstance(self.container, fields.Nested) or type(self.container) is fields.Raw
def is_attr(val):
return self.container.attribute and hasattr(val, self.container.attribute)
# Put 'data' as key before the list, and return the dict
return {'data': [
self.container.output(idx,
val if (isinstance(val, dict) or is_attr(val)) and not is_nested else value)
for idx, val in enumerate(value)
]}
def output(self, key, data, ordered=False, **kwargs):
value = fields.get_value(key if self.attribute is None else self.attribute, data)
if fields.is_indexable_but_not_string(value) and not isinstance(value, dict):
return self.format(value)
if value is None:
return self._v('default')
return [marshal(value, self.container.nested)]
def schema(self):
schema = super(ListData, self).schema()
schema.update(minItems=self._v('min_items'),
maxItems=self._v('max_items'),
uniqueItems=self._v('unique'))
# work around to get the documentation as I want
schema['type'] = 'object'
schema['properties'] = {}
schema['properties']['data'] = {}
schema['properties']['data']['type'] = 'array'
schema['properties']['data']['items'] = self.container.__schema__
return schema
This is the structure of my json file
},
"client1": {
"description": "blabla",
"contact name": "",
"contact email": "",
"third party organisation": "",
"third party contact name": "",
"third party contact email": "",
"ranges": [
"1.1.1.1",
"2.2.2.2",
"3.3.3.3"
]
},
"client2": {
"description": "blabla",
"contact name": "",
"contact email": "",
"third party organisation": "",
"third party contact name": "",
"third party contact email": "",
"ranges": [
"4.4.4.4",
"2.2.2.2"
]
},
I've seen ways to export specific parts of this json file but not everything. Basically all I want to do is search through the file using user input.
All I'm struggling with is how I actually use the user input to search and print everything under either client1 or client2 based on the input? I am sure this is only 1 or 2 lines of code but cannot figure it out. New to python. This is my code
data = json.load(open('clients.json'))
def client():
searchq = input('Client to export: '.capitalize())
search = ('""'+searchq+'"')
a = open('Log.json', 'a+')
a.write('Client: \n')
client()
This should get you going:
# Safely open the file and load the data into a dictionary
with open('clients.json', 'rt') as dfile:
data = json.load(dfile)
# Ask for the name of the client
query = input('Client to export: ')
# Show the corresponding entry if it exists,
# otherwise show a message
print(data.get(query, 'Not found'))
I'm going to preface this by saying this is 100% a drive-by answering, but one thing you could do is have your user use a . (dot) delimited format for specifying the 'path' to the key in the dictionary/json structure, then implementing a recursive function to seek out the value under that path like so:
def get(query='', default=None, fragment=None):
"""
Recursive function which returns the value of the terminal
key of the query string supplied, or if no query
is supplied returns the whole fragment (dict).
Query string should take the form: 'each.item.is.a.key', allowing
the user to retrieve the value of a key nested within the fragment to
an arbitrary depth.
:param query: String representation of the path to the key for which
the value should be retrieved
:param default: If default is specified, returns instead of None if query is invalid
:param fragment: The dictionary to inspect
:return: value of the specified key or fragment if no query is supplied
"""
if not query:
return fragment
query = query.split('.')
try:
key = query.pop(0)
try:
if isinstance(fragment, dict) and fragment:
key = int(key) if isinstance(fragment.keys()[0], int) else key
else:
key = int(key)
except ValueError:
pass
fragment = fragment[key]
query = '.'.join(query)
except (IndexError, KeyError) as e:
return default if default is not None else None
if not fragment:
return fragment
return get(query=query, default=default, fragment=fragment)
There are going to be a million people who come by here with better suggestions than this and there are doubtless many improvements to be made to this function as well, but since I had it lying around I thought I'd put it here, at least as a starting point for you.
Note:
Fragment should probably be made a positional argument or something. IDK. Its not because I had to rip some application specific context out (it used to have a sensible default state) and I didn't want to start re-writing stuff, so I leave that up to you.
You can do some cool stuff with this function, given some data:
d = {
'woofage': 1,
'woofalot': 2,
'wooftastic': ('woof1', 'woof2', 'woof3'),
'woofgeddon': {
'woofvengers': 'infinity woof'
}
}
Try these:
get(fragment=d, query='woofage')
get(fragment=d, query='wooftastic')
get(fragment=d, query='wooftastic.0')
get(fragment=d, query='woofgeddon.woofvengers')
get(fragment=d, query='woofalistic', default='ultraWOOF')
Bon voyage!
Pass the json format into Dict then look into the topic you want and Read or write it
import json
r = {'is_claimed': True, 'rating': 3.5}
r = json.dumps(r) # Here you have json format {"is_claimed": true, "rating": 3.5}
Json to Dict:
loaded_r = json.loads(r) # {'is_claimed': True, 'rating': 3.5}
print (r)#Print json format
print (loaded_r) #Print dict
Read the Topic
Data=loaded_r['is_claimed'] #Print Topic desired
print(Data) #True
Overwrite the topic
loaded_r['is_claimed']=False
And also this would do the same
print(loaded_r['client1']['description'])
Input/Goal
My input data is an OrderedDict for which there can be a variable depth of nested OrderedDicts so I have opted to handle parsing this output recursively. The desired output is a csv with header.
Elaboration of Problem
My code below will work once I am able to correctly define field_name upon traversing back up a branch after completing all of a branch's leaves. (i.e. Type_1.Field_3.Data will incorrectly be called Type_1.Field_2.Field_3.Data).
Once the leaves on a branch have been exhausted, I want to remove the last .Field_x from the field_name so that a new (correct) one can be added for the following object.
Request for Help
Does anyone see where I can include this feature? Thanks,
...
Dependencies:
Code Snippet:
def get_soql_fields(soql):
soql_fields = re.search('(?<=select)(?s)(.*)(?=from)', soql) # get fields
soql_fields = re.sub(' ', '', soql_fields.group()) # remove extra spaces
fields = re.split(',|\n|\r', soql_fields) # split on commas and newlines
fields = [field for field in fields if field != ''] # remove empty strings
return fields
def parse_output(data, soql):
fields = get_soql_fields(soql)
header = fields
master = [header]
for record in data['records']: # for each 'record' in response
row = []
for obj, value in record.iteritems(): # for each obj in record
if isinstance(value, basestring): # if query base object has desired fields
if obj in fields:
row.append(value)
elif isinstance(value, dict): # traverse down into object
path = obj
row.append(_traverse_output(obj, value, fields, row, path))
master.append(row)
return master
def _traverse_output(obj, value, fields, row, path):
for f, v in value.iteritems(): # for each item in obj
if not isinstance(v, (dict, list, tuple)):
field_name = '{path}.{name}'.format(path=path, name=f) # TODO fix this to full field name
print('FName: {0}'.format(field_name))
if field_name in fields:
print('match')
row.append(v)
elif isinstance(v, dict): # it is a dict
path += '.{obj}'.format(obj=f)
_traverse_output(f, v, fields, row, path)
Example Salesforce SOQL:
select
Type_1.Field_1,
Type_1.Field_2.Data,
Type_1.Field_3,
Type_1.Field_4,
Type_1.Field_5.Data_1.Data,
Type_1.Field_6,
Type_2.Field_1,
Type_2.Field_2
from
Obj_1
limit
1
;
Example Salesforce Output:
{
"records": [
{
"attributes": {
"type": "Obj_1",
"url": "<url>"
},
"Type_1": {
"attributes": {
"type": "Type_1",
"url": "<url>"
},
"Field_1": "<stuff>",
"Field_2": {
"attributes": {
"type": "Field_2",
"url": "<url>"
},
"Data": "<data>"
},
"Field_3": "<data>",
"Field_4": "<data>",
"Field_5": {
"attributes": {
"type": "Field_2",
"url": "<url>"
},
"Data_1": {
"attributes": {
"type": "Data_1",
"url": "<url>"
},
"Data": "<data>"
}
},
"Field_6": 1.0
},
"Type_2": {
"attributes": {
"type": "Type_2",
"url": "<url>"
},
"Field_1": "<data>",
"Field_2": "<data>"
}
}
]
}
I worked out a quick solution for this. I'll just note what I figured out, and append the code I wrote to the end.
Essentially your problem is that you keep trying to modify path in place, which isn't going to work. Instead do something like
new_path = path + '.{obj}'.format(obj=f)
_traverse_output(f, v, fields, row, new_path)
A note about this: it will NOT necessarily result in a row where the values are in the same order as the header (i.e., if Type_1.Field_1 is in position 0 of the header list, then the value corresponding to it might not be).
The easy way to solve this (and handle csvs in general) is to use DictWriter from the csv module, then pass an empty dictionary to your first call where the keys will be the field names and the values will be their values.
Another way to solve the problem is to pre-populate your row list with None or empty strings, then use the list.index method to assign the value to the appropriate position.
I wrote an implementation of _traverse_output as examples for each, though they differ slightly from your code. They take an element of the 'records' list.
Dictionary Example
def _traverse_output_with_dict(record, fields, row_values, field_name=''):
for obj, value in record.iteritems():
new_field_name = '{}.{}'.format(field_name, obj) if field_name else obj
print new_field_name
if not isinstance(value, dict):
if new_field_name in fields:
row_values[new_field_name] = value
else:
_traverse_output_with_dict(value, fields, row_values, new_field_name)
List Example
def _traverse_output_with_list(record, fields, row, field_name=''):
while len(row) < len(fields):
row.append('')
for obj, value in record.iteritems():
new_field_name = '{}.{}'.format(field_name, obj) if field_name else obj
print new_field_name
if not isinstance(value, dict):
if new_field_name in fields:
row[fields.index(new_field_name)] = value
else:
_traverse_output_with_list(value, fields, row, new_field_name)