We Keep Coding

ValueError: not enough values to unpack while running unit tests Django ModelViewSet

Am testing an endpoint that retrieves data using a ModelViewSet, and am passing a param via a URL to it to get data but am getting this error when I run the unit tests:
File "/Users/lutaayaidris/Documents/workspace/project_sample/project_sample/financing_settings/tests.py", line 195, in test_get_blocks
self.block_get_data), content_type='application/json')
File "/Users/lutaayaidris/Documents/workspace/project_sample/lib/python3.6/site-packages/rest_framework/test.py", line 286, in get
response = super().get(path, data=data, **extra)
File "/Users/lutaayaidris/Documents/workspace/project_sample/lib/python3.6/site-packages/rest_framework/test.py", line 194, in get
'QUERY_STRING': urlencode(data or {}, doseq=True),
File "/Users/lutaayaidris/Documents/workspace/project_sample/lib/python3.6/site-packages/django/utils/http.py", line 93, in urlencode
for key, value in query:
ValueError: not enough values to unpack (expected 2, got 1)
This is how I have structured my tests , plus some dummy data for testing :
class TemplateData:
"""Template Mock data."""
step_get_data = {
"param": "step"
}
block_get_data = {
"param": "block"
}
get_no_data = {
"param_": "block"
}
class TemplateViewTests(TestCase, TemplateData):
"""Template Tests (Block & Step)."""
def setUp(self):
"""
Initialize client, Step and Block id and data created.
"""
self.client = APIClient()
self.block_id = 0
self.step_id = 0
self.create_block_step_data()
def create_block_step_data(self):
"""Create ProcessVersion, Step, & Block mock data."""
self.process_version = ProcessVersion.objects.create(
tag="TESTING_TAG",
is_process_template=False,
status="IN EDITING",
attr_map="TESTING_ATTR",
loan_options=None
)
self.step = Step.objects.create(
version=self.process_version,
is_process_template=True,
title="TESTING",
help_text="TESTING",
order=1,
slug="slug",
can_be_duplicated=False,
max_duplicated_number=2,
)
self.step_id = self.step.pk
self.block_id = Block.objects.create(
step=self.step,
is_process_template=True,
title="TESTING",
information_text="This is testing "
"information",
order=1,
depending_field="depending_field",
visibility_value="visibility_value",
slug="slug",
can_be_duplicated=False,
max_duplicated_number=2,
).pk
self.process_version_1 = ProcessVersion.objects.create(
tag="TESTING_TAG",
is_process_template=False,
status="IN EDITING",
attr_map="TESTING_ATTR",
loan_options=None
)
self.step_1 = Step.objects.create(
version=self.process_version_1,
is_process_template=True,
title="TESTING",
help_text="TESTING",
order=1,
slug="slug",
can_be_duplicated=False,
max_duplicated_number=2,
)
self.block_1 = Block.objects.create(
step=self.step,
is_process_template=True,
title="TESTING",
information_text="This is testing "
"information",
order=1,
depending_field="depending_field",
visibility_value="visibility_value",
slug="slug",
can_be_duplicated=False,
max_duplicated_number=2,
).pk
def test_get_blocks(self):
"""Test get list of Block. """
response = self.client.get(
"/api/v1/financing-settings/template/",
data=json.dumps(
self.block_get_data), content_type='application/json')
self.assertEqual(response.status_code, 200)
def test_get_steps(self):
"""Test get list of Step. """
response = self.client.get(
"/api/v1/financing-settings/template/",
data=json.dumps(
self.block_get_data),
content_type='application/json')
self.assertEqual(response.status_code, 200)
def test_no_step_or_block(self):
"""Test get no list of Step or Block. """
response = self.client.get(
"/api/v1/financing-settings/template/",
data=json.dumps(
self.block_get_data),
content_type='application/json')
self.assertEqual(response.status_code, 204)
As you can see above those are my tests, I have already setup the data , now I want to retrieve back the data, but because of the exception above I can't.
Lastly, in my endpoint implementation, I used a Viewset to handle this , below is the code :
class TemplateView(ModelViewSet):
"""ViewSet for Saving Block/ Step template."""
def list(self, request, *args, **kwargs):
"""Get list of Block/Steps with is_process_template is equal to True."""
param = request.data['param']
if param == "block":
_block = Block.objects.filter(is_process_template=True).values()
return JsonResponse({"data": list(_block)}, safe=False, status=200)
elif param == "step":
_step = Step.objects.filter(is_process_template=True)
return JsonResponse({"data": list(_step)}, safe=False, status=200)
return Response(status=status.HTTP_204_NO_CONTENT)
What is causing this , in my understanding I feel like everything should work.

The function Client.get expect a dictionary as data argument and try to encode it in the url using the function urlencode. You could do something like that:
from django.test import Client
c = Client()
block_get_data = {
"param": "block"
}
c.get('path', block_get_data)
block_get_data will be sent in the url as 'param=block'
If you want to send JSON formated data in a GET method, you can use Client.generic function as follow:
from django.test import Client
import json
c = Client()
block_get_data = {
"param": "block"
}
c.generic('GET', 'path', json.dumps(block_get_data), 'application/json')

You are facing this error because this dict
block_get_data = {
"param": "block"
}
you are trying to use it in this way
for key,val in block_get_data
and it will produce the error like
for key,val in block_get_data:
ValueError: too many values to unpack (expected 2)
It will be solved if your loop through dict by using .items() method.
for key,val in block_get_data.items():
I think by passing parameter as self.block_get_data.items() may solve your problem.

Ajax, Flask route, data appended to URL, server error

I'm using a Flask route to update a MYSQL database and I use an AJAX call to trigger this event. Whenever I run the webpage, I get an internal server error because it appends ?bet=1 to my url [and the is not a defined url]. I don't really get what I have to do to make this not happen and it to just run the normal route as defined in my python file. I've read a lot, but I'm not a programmer [I'm a scientist] and I don't know why this is happening. I think that when I submit the AJAX url it automatically updates the url with the data. Thanks for you help.
Custom.py route that I use with Flask
#custom_code.route('/saveddd', methods=['POST','GET'])
def savdict():
# get form data from ajax
#bet = request.form['bet']
try:
bet = request.form['bet']
db_url = "mysql+pymysql://root:Password!#127.0.0.1:3306/mydatabase"
table_name = 'my_experiment_table'
data_column_name = 'datastring'
# boilerplace sqlalchemy setup
# boilerplace sqlalchemy setup
engine = create_engine(db_url, echo=True)#altered by adding echo
conn=engine.connect()
metadata = MetaData()
metadata.bind = engine
existingtable2 = Table('dictatordecisions', metadata, autoload=True)
#print(existingtable2)
#print ("1")
insort = existingtable2.insert().values(Bet = bet)
#fire = sqlalchemy.sql.expression.insert(existingtable2, values=.5, inline=False, bind=None, prefixes=None, returning=None, return_defaults=False)
conn.execute(insort)
return "cool it worked"
except TemplateNotFound:
abort(404)
AJAX call
function death6(){
//var dictoatorresult = 0
//var selfammount = sliderfoo.value;
if (sliderfoo.value == "") {
alert ("please complete the task")
} else {dictatorchoice = sliderfoo.value;
psiTurk.recordTrialData({'phase':'DictatorGame','status':'submit','choice':dictatorchoice});
psiTurk.recordUnstructuredData('dictatordecision', dictatorchoice);
psiTurk.recordUnstructuredData('dice rolls purchased', purchase);
psiTurk.recordUnstructuredData('race manipulation if 1, plant if 2', condition);
psiTurk.recordUnstructuredData('race answers', RaceAnswers);
psiTurk.saveData();
$.ajax({})
$.ajax({
cache: true,
Type:'POST',
url:'/saveddd',
async: false,
data :{"bet" : dictatorchoice},
//crossDomain: true,
//async: false,
dataType: 'text'
//processData: false,
//beforeSend: function (xhr) {
//xhr.setRequestHeader("x-ajax-call", "no-cache");
})
;
psiTurk.showPage('postquestionnaire.html')
}
//d3.selectAll("input").remove();
//d3.selectAll("input").remove();
//d3.selectAll("div").remove();
}
Button maker
function makeButton2(text, callback){
d3.select("body")
.insert("p")
.insert("p")
d3.select("body")
.insert("button")
.attr("type", "button")
.attr("class", "btn btn-primary btn-lg")
.text(text)
.on("click", function(d){callback();});
}
Sliderfoo defined
var sliderSlide = document.createElement("input");
sliderSlide.setAttribute("type","number");
sliderSlide.setAttribute("fontsize","500px");
sliderSlide.setAttribute("min","0.00");
sliderSlide.setAttribute("max","1.00");
sliderSlide.setAttribute("step","0.01");
//sliderSlide.setAttribute("placeholder","0.50");
sliderSlide.setAttribute("id","sliderfoo");
sliderSlide.setAttribute("style", "font-size:25px")
//sliderSlide.setAttribute("syle","width: 300px");
//sliderSlide.setAttribute("width","400")

?bet=1 means you send it as GET, not POST.
Problem can be bacuse you used Type: 'POST' but it should be method: 'POST'.
You could also use directly $.post() instead of $.ajax()

Flask request.files is always empty

I am making a POST request to send a JSON object with keys containing files. An example of what I send to the backend is:
export interface PosInputFiles {
org: string;
year_month: string;
in_master_file: File;
iv_file?: File;
sales_file?: File;
recv_file?: File;
transfer_file?: File;
adjust_file?: File;
pcount_file?: File;
gift_file?: File;
xrate?: string;
}
My POST request looks like:
generateMaster(args: PosInputFiles) {
return this.http.post('http://localhost:5000/api', args, { headers: this.headers});
}
When I try to access these files from request.json, the values are an empty dict ({}).
try:
org = request.json['org']
year_month = request.json['year_month']
in_master_file = request.json['in_master_file']
iv_file = None if 'iv_file' not in request.json else request.json['iv_file']
sales_file = None if 'sales_file' not in request.json else request.json['sales_file']
recv_file = None if 'recv_file' not in request.json else request.json['recv_file']
transfer_file = None if 'transfer_file' not in request.json else request.json['transfer_file']
adjust_file = None if 'adjust_file' not in request.json else request.json['adjust_file']
pcount_file = None if 'pcount_file' not in request.json else request.json['pcount_file']
gift_file = None if 'gift_file' not in request.json else request.json['gift_file']
xrate = None if 'xrate' not in request.json else request.json['xrate']
except:
return { "post" : "failed" }
print(in_master_file)
print(len(request.files))
return { "post": "success"}
Then I tried sending only one file and made sure len(request.json) == 0 through POSTMan and my frontend (Angular8). However, len(request.files) is also 0 and every time I try to access something, there is 400 error. My POST request is successful as I always print {"post", "success"} but for some reason, no files make it to the backend. All my files sent are real files and I have made sure that I am sending the file. Thank you so much for your help!

For those who might have the same problem eventually, here's how I solved this issue. Flask doesn't recognize files that aren't of FormData type so that's why I could only access JSON. Thus, I had to append all my files to a FormData variable.
generateMaster(submittedFiles: PosInputFiles) {
const formData: FormData = new FormData();
Object.keys(submittedFiles).forEach(key => {
if (submittedFiles[key] instanceof File) {
formData.append(key, submittedFiles[key], submittedFiles[key].name);
} else {
formData.append(key, new File([], submittedFiles[key]), submittedFiles[key].name);
}
})
return this.http.post('http://localhost:5000/api', formData, { headers: this.headers});
}
Then backend will finally recognize the files. In order to get other string data, and floats, I stored those values as the filename and could access them as such.
def post(self):
# Initilize arguments
org = request.files.get('org').filename
year_month = request.files.get('year_month').filename
in_master_file = request.files.get('in_master_file')
iv_file = request.files.get('iv_file')
sales_file = request.files.get('sales_file')
...
xrate = None if 'xrate' not in request.files else float(request.files.get('xrate').filename)

JSON data not reaching view pyramid

I'm trying to build a page where when the user presses a button a variable which initially is 0 increments with 1. This number is then sent asynchronously to the server by using jQuery AJAX.
What I have so far is:
In my __init__.py file:
def main(global_config, **settings):
engine = engine_from_config(settings, 'sqlalchemy.')
DBSession.configure(bind = engine)
Base.metadata.bind = engine
config = Configurator(settings = settings)
config.include('pyramid_jinja2')
config.add_static_view('static', 'static')
config.add_static_view('scripts', 'scripts')
# Removed the other views
config.add_route("declare_usage", '/user/{user_id}/{address_id}/declare')
config.add_route("declare_usage_json",'/user/{user_id}/{address_id}/declare.json')
config.scan()
My HTML + Jinja2:
#Removed code for simplicity
<div id="button_add">Add</div>
{{val}}
My JS:
$(document).ready(function(){
var room = 0;
jQuery.ajax({type:'POST',
url: '/user/1/5/declare', #I use a direct user ID and a direct address ID as I'm not sure how to send this to JS from Pyramid ... yet :).
data: JSON.stringify(room),
contentType: 'application/json; charset=utf-8'});
$('#button_add').click(function(){
room = room + 1;
});
});
My view code:
#view_config(route_name = 'declare_usage', renderer = 'declara.jinja2')
#view_config(route_name = 'declare_usage_json', renderer = 'json')
def declara_consum(request):
#Removed code for simplicity
val = request.POST.get('room') #I get a "None value in my html" if I change to request.json_body -> I get an error that there is no json to be parsed.
return { 'val' : val }
What happens is that when I open the debugger the POST request is successful with no data and on the page I get 2 options for 'val':
None -> When I use val = request.POST.get('room')
Error ValueError: No JSON object could be decoded -> When I use val = request.json_body
Also, still can't get it to work if in my JS i change url to be /user/1/5/declare.json and/or data to {'room' : room}
Can somebody please point out what I'm doing wrong?

you don't need another route declare_usage_json, just need separate two function like this
#view_config(route_name = 'declare_usage', renderer = 'declara.jinja2')
def declara_consum(request):
# this will response to your jinja2
return { 'val' : val }
#view_config(route_name = 'declare_usage', xhr=True, renderer = 'json')
def declara_consum_ajax(request):
# this will response to your asynchronously request
val = request.POST.get('room')
return { 'val' : val }
when you send a request using ajax, this will goto the second function.
$.ajax({
type: 'POST',
url: '/user/1/5/declare',
data: {'room' : room},
dataType: 'json'
}).done(function(response){
// update your data at html
});

DataRequired validator is broken for wtforms.BooleanField

I'm using WTForms (together with Flask, flask-wtf, sqlalchemy) to validate incoming JSON for REST APIs. I realize that WTForms is aimed more towards HTML form rendering and validation, but I chose it because it can autogenerate forms out of my sqlalchemy models (thanks to wtforms.ext.sqlalchemy).
Anyway, here is the problem. One of my models includes boolean field which translates to wtforms.BooleanField with DataRequired validator. The issue is that validation fails with 'This field is required' error message even if I pass correct data. My Form:
class MyForm(Form):
name = TextField('name', validators=[DataRequired()])
disabled = BooleanField('disabled', validators=[DataRequired()])
JSON data is like this:
'{"name": "John", "disabled": "false"}'
What I'm expecting:
{"disabled": "false"} -> validates successful, coerced Python data: {'disabled': False}
{"disabled": "true"} -> validates successful, coerced Python data: {'disabled': True}
{"disabled": ""} or '{"disabled": "foo"}' -> fails validation
Currently in first case validation is failed with {'disabled': [u'This field is required.']}
I know there is a note in docs that says DataRequired validator "require coerced data, not input data", but 1) the form is autogenerated by wtforms.ext.sqlalchemy and 2) how it is supposed to behave if I use InputRequired validator? Check (via form.validate()) that some data exists and then check that this data is "true" or "false"?
To summarize, my question is:
What is the correct way of validating wtforms.BooleanField?
Maybe there is some other framework that can validate incoming JSON against given sqlalchemy models?
Thanks.

There are a number of ways of going about this. You could write your own converter to make of use a radiofield with true/false choices, you could use a data filter, you could set a default value, but I think the behavior you want will be possible with this:
MyForm = model_form(MyModel, db_session=db, field_args = {
'disabled' : {
'false_values': ['false'],
'validators' : [InputRequired()] }
})
EDIT: If you wanted a stricter handler you could do the following:
class BooleanRequired(object):
field_flags = ('required', )
def __init__(self, message=None):
self.message = message
def __call__(self, form, field):
if field.data is None:
if self.message is None:
message = field.gettext('This field is required.')
else:
message = self.message
field.errors[:] = []
raise StopValidation(message)
class StrictBooleanField(BooleanField):
def process_formdata(self, valuelist):
self.data = None
if valuelist:
if valuelist[0] == 'false':
self.data = False
elif valuelist[0] == 'true':
self.data = True
class StrictModelConverter(ModelConverter):
#converts('Boolean')
def conv_Boolean(self, field_args, **extra):
return StrictBooleanField(**field_args)
MyForm = model_form(MyModel, db_session=db, converter=StrictModelConverter(),
field_args = { 'disabled' : { 'validators': [BooleanRequired()] }
})