We Keep Coding

RequestValidationError: JSON object must be str, bytes or bytearray (type=type_error.json)

I've got an endpoint in my FastAPI application for receiving form data:
#router.post("/foobar")
async def handler(
form_data: Bar = Depends(Bar.as_form),
) -> JSONResponse:
...
What I'm trying to do is to validate form data with help of pydantic. Here are the models:
from fastapi import Form
from pydantic import BaseModel, Json
class Foo(BaseModel):
a: str
class Bar(BaseModel):
any_field: Optional[List[Foo]]
#classmethod
def as_form(
cls,
any_field: Json[List[Foo]] = Form(None, media_type="application/json"),
) -> "Bar":
return cls(any_field=any_field)
But I'm getting the following error:
fastapi.exceptions.RequestValidationError: 1 validation error for Request
body -> any_field
JSON object must be str, bytes or bytearray (type=type_error.json)
I've added exception handler for RequestValidationError to make sure that any_field is actually str type:
#application.exception_handler(RequestValidationError)
async def validation_exception_handler(request: Request, exc: RequestValidationError):
body: FormData = exc.body
return JSONResponse(
content={"msg": str([type(v) for v in body.values()])},
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY
)
Here is my request:
As you may see, it is str.
Why does this error occur?

You could let FastAPI do the validation on its own. For example...
from fastapi import FastAPI, Form, File, UploadFile
from pydantic import BaseModel, Json
from typing import Optional, List
class Foo(BaseModel):
a: str
class Bar(BaseModel):
any_field: Optional[List[Foo]] = Form()
app = FastAPI()
#app.post("/foobar")
async def handler(upload: UploadFile = File(), form_data: Json[Bar] = Form()):
return {'form_data': form_data, 'upload': upload.filename}
Here is code illustrating use of the endpoint...
import requests
import json
reqUrl = "http://127.0.0.1:8000/foobar"
with open("/Users/pelletier/Playground/scores.csv", "rb") as upload:
post_files = {'upload': upload}
payload = {'form_data': json.dumps({'any_field': [{'a': 'foo'}, {'a': 'foo'}]})}
response = requests.post(reqUrl, data=payload, files=post_files)
print(response.text)

How can define a pytest function to test an endpoint using FastAPI with a pydantic model that has a field of type datetime

I have a pytest function to test a FastAPI endpoint, this is just to verify that right data can create an entity, here the test function:
import pytest
from databases import Database
from fastapi import FastAPI, status
from httpx import AsyncClient
from modules.member.members_schemas import MemberCreate
pytestmark = pytest.mark.asyncio
class TestCreateMember:
async def test valid_input_creates_member(self, app: FastAPI, client: AsyncClient,
db: Database) -> None:
new_member = MemberCreate(
fullname="John Smith"
dni="123456789",
birthdate=datetime(1985, 10, 4),
email="testemail#test.com",
)
res = await client.post(
app.url_path_for("members:create-member), json={"member": new_member.dict()}
)
assert res.status_code == status.HTTP_201_CREATED
Here the pydantic model (schema in my case):
from datetime import datetime
from pydantic import BaseModel, BaseConfig, EmailStr
class BaseSchema(BaseModel):
class Config(BaseConfig):
allow_population_by_field_name = True
orm_mode = True
class MemberCreate(BaseSchema):
fullname: str
dni: str
birth_date: datetime
email: EmailStr
And here is the endpoint to test:
from databases import Database
from fastapi import APIRouter, Body, Depends, status
router = APIRouter(
prefix="/members",
tags=["members"],
responses={404: {"description": "Not found"}},
)
#router.post(
"/",
response_model=MemberPublic,
name="members:create-member",
status_code=status.HTTP_201_CREATED,
)
async def create_member(
member: MemberCreate = Body(..., embed=True),
db: Database = Depends(get_database),
current_user: UserInDB = Depends(get_current_active_user),
) -> ServiceResult:
result = await MemberService(db).create_member(member, current_user)
return handle_result(result)
ServiceResult type and handle_result() function are funcionalies to make a stardard answer from each endpoint made it, not problem working with it with a lot of other endpoints. When I ran pytest on this particular test, I got this error:
self = <json.encoder.JSONEncoder object at 0x7fe1ebe3fd10>
o = datetime.datetime(1985, 10, 4, 0, 0)
def default(self, o):
"""Implement this method in a subclass such that it returns
a serializable object for ``o``, or calls the base implementation
(to raise a ``TypeError``).
For example, to support arbitrary iterators, you could
implement default like this::
def default(self, o):
try:
iterable = iter(o)
except TypeError:
pass
else:
return list(iterable)
# Let the base class default method raise the TypeError
return JSONEncoder.default(self, o)
"""
> raise TypeError(f'Object of type {o.__class__.__name__} '
f'is not JSON serializable')
E TypeError: Object of type datetime is not JSON serializable
So I changed the test adding a json.dumps() (importing json, of course), this way:
async def test valid_input_creates_member(self, app: FastAPI, client: AsyncClient,
db: Database) -> None:
new_member = MemberCreate(
fullname="John Smith"
dni="123456789",
birthdate=datetime(1985, 10, 4),
email="testemail#test.com",
)
new_member_json = json.dumps(new_member.dict(), indent=4, sort_keys=True, default=str)
res = await client.post(
app.url_path_for("members:create-member), json={"member": new_member_json}
)
assert res.status_code == status.HTTP_201_CREATED
After I ran pytest again, I got this error new error:
> assert res.status_code == status.HTTP_201_CREATED
E assert 422 == 201
E + where 422 = <Response [422 Unprocessable Entity]>.status_code
E + and 201 = status.HTTP_201_CREATED
Which is a Pydantic validation error at the endpoint.
When I see the endpoint post in the swagger view, the datetime field is presented as a string, so no problems to be serialized, but I don´t want to change the schema to receive a str, because I would lose the pydantic's power. So my question is, How can I set the test in order to override the pydantc validation and accept the datetime field? or I must change the schema to have a string field and process it internally in order to receive only valid datetime data?. I'll appreciate any help, because I'm stuck on this problem for several hours.

As suggets matslindh, I changed the test function, eliminating the pydantic object and it was substituted by a simple dict, this way:
class TestCreateMember:
async def test valid_input_creates_member(self, app: FastAPI, client: AsyncClient,
db: Database) -> None:
new_member = {
"fullname": "John Smith"
"dni": "123456789",
"birthdate": "1985-10-04T00:00",
"email": "testemail#test.com",
}
res = await client.post(
app.url_path_for("members:create-member), json={"member": new_member}
)
assert res.status_code == status.HTTP_201_CREATED
After that everything works as expected

How to allow List as query params instead of requestbody in pydantic model for fastapi

So I have a simple fastapi model as follows:
from typing import List
from fastapi import Query, Depends, FastAPI
from pydantic import BaseModel
class QueryParams(BaseModel):
req1: float = Query(...)
opt1: int = Query(None)
req_list: List[str] = Query(...)
app = FastAPI()
#app.post("/test", response_model=QueryParams)
def foo(q: QueryParams = Depends()):
return q
which works with the following command: curl -X "POST" "http://localhost:8000/test?req1=1" -d '{["foo"]}'
However! I need it to work additionally with allowing the params in the uri request as such:
curl -X "POST" "http://localhost:8000/test?req1=1&req_list=foo"
I know that if I took out the req_list from the BaseModel, and shoved it in the function header as such
from typing import List
from fastapi import Query, Depends, FastAPI
from pydantic import BaseModel
class QueryParams(BaseModel):
req1: float = Query(...)
opt1: int = Query(None)
app = FastAPI()
#app.post("/test", response_model=QueryParams)
def foo(q: QueryParams = Depends(), req_list: List[str] = Query(...)):
return q
that it would work, but is there any way to keep it in the basemodel?

Well I figured it out:
from typing import List
from fastapi import Query, Depends, FastAPI
from pydantic.dataclasses import dataclass
#dataclass
class QueryParams:
req1: float = Query(...)
opt1: int = Query(None)
req_list: List[str] = Query(...)
app = FastAPI()
#app.post("/test", response_model=QueryParams)
def foo(q: QueryParams = Depends()):
return q

Testing in FastAPI using Tortoise-ORM

I'm trying to write some async tests in FastAPI using Tortoise ORM under Python 3.8 but I keep getting the same errors (seen at the end). I've been trying to figure this out for the past few days but somehow all my recent efforts in creating tests have been unsuccessful.
I'm following the fastapi docs and tortoise docs on this one.
main.py
# UserPy is a pydantic model
#app.post('/testpost')
async def world(user: UserPy) -> UserPy:
await User.create(**user.dict())
# Just returns the user model
return user
simple_test.py
from fastapi.testclient import TestClient
from httpx import AsyncClient
#pytest.fixture
def client1():
with TestClient(app) as tc:
yield tc
#pytest.fixture
def client2():
initializer(DATABASE_MODELS, DATABASE_URL)
with TestClient(app) as tc:
yield tc
finalizer()
#pytest.fixture
def event_loop(client2): # Been using client1 and client2 on this
yield client2.task.get_loop()
# The test
#pytest.mark.asyncio
def test_testpost(client2, event_loop):
name, age = ['sam', 99]
data = json.dumps(dict(username=name, age=age))
res = client2.post('/testpost', data=data)
assert res.status_code == 200
# Sample query
async def getx(id):
return await User.get(pk=id)
x = event_loop.run_until_complete(getx(123))
assert x.id == 123
# end of code
My errors vary on whether I'm usinng client1 or client2
Using client1 error
RuntimeError: Task <Task pending name='Task-9' coro=<TestClient.wait_shutdown() running at <my virtualenv path>/site-packages/starlette/testclient.py:487> cb=[_run_until_complete_cb() at /usr/lib/python3.8/asyncio/base_events.py:184]> got Future <Future pending> attached to a different loop
Using client2 error
asyncpg.exceptions.ObjectInUseError: cannot drop the currently open database
Oh, I've also tried using httpx.AsyncClient but still no success (and more errors). Any ideas because I'm out of my own.

It cost me about one hour to make the async test worked. Here is the example:
(Python3.8+ is required)
conftest.py
import pytest
from httpx import AsyncClient
from tortoise import Tortoise
from main import app
DB_URL = "sqlite://:memory:"
async def init_db(db_url, create_db: bool = False, schemas: bool = False) -> None:
"""Initial database connection"""
await Tortoise.init(
db_url=db_url, modules={"models": ["models"]}, _create_db=create_db
)
if create_db:
print(f"Database created! {db_url = }")
if schemas:
await Tortoise.generate_schemas()
print("Success to generate schemas")
async def init(db_url: str = DB_URL):
await init_db(db_url, True, True)
#pytest.fixture(scope="session")
def anyio_backend():
return "asyncio"
#pytest.fixture(scope="session")
async def client():
async with AsyncClient(app=app, base_url="http://test") as client:
print("Client is ready")
yield client
#pytest.fixture(scope="session", autouse=True)
async def initialize_tests():
await init()
yield
await Tortoise._drop_databases()
settings.py
import os
from dotenv import load_dotenv
load_dotenv()
DB_NAME = "async_test"
DB_URL = os.getenv(
"APP_DB_URL", f"postgres://postgres:postgres#127.0.0.1:5432/{DB_NAME}"
)
ALLOW_ORIGINS = [
"http://localhost",
"http://localhost:8080",
"http://localhost:8000",
"https://example.com",
]
main.py
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from models.users import User, User_Pydantic, User_Pydantic_List, UserIn_Pydantic
from settings import ALLOW_ORIGINS, DB_URL
from tortoise.contrib.fastapi import register_tortoise
app = FastAPI()
app.add_middleware(
CORSMiddleware,
allow_origins=ALLOW_ORIGINS,
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
#app.post("/testpost", response_model=User_Pydantic)
async def world(user: UserIn_Pydantic):
return await User.create(**user.dict())
#app.get("/users", response_model=User_Pydantic_List)
async def user_list():
return await User.all()
register_tortoise(
app,
config={
"connections": {"default": DB_URL},
"apps": {"models": {"models": ["models"]}},
"use_tz": True,
"timezone": "Asia/Shanghai",
"generate_schemas": True,
},
)
models/base.py
from typing import List, Set, Tuple, Union
from tortoise import fields, models
from tortoise.queryset import Q, QuerySet
def reduce_query_filters(args: Tuple[Q, ...]) -> Set:
fields = set()
for q in args:
fields |= set(q.filters)
c: Union[List[Q], Tuple[Q, ...]] = q.children
while c:
_c: List[Q] = []
for i in c:
fields |= set(i.filters)
_c += list(i.children)
c = _c
return fields
class AbsModel(models.Model):
id = fields.IntField(pk=True)
created_at = fields.DatetimeField(auto_now_add=True, description="Created At")
updated_at = fields.DatetimeField(auto_now=True, description="Updated At")
is_deleted = fields.BooleanField(default=False, description="Mark as Deleted")
class Meta:
abstract = True
ordering = ("-id",)
#classmethod
def filter(cls, *args, **kwargs) -> QuerySet:
field = "is_deleted"
if not args or (field not in reduce_query_filters(args)):
kwargs.setdefault(field, False)
return super().filter(*args, **kwargs)
class PydanticMeta:
exclude = ("created_at", "updated_at", "is_deleted")
def __repr__(self):
return f"<{self.__class__.__name__} {self.id}>"
models/users.py
from tortoise.contrib.pydantic import pydantic_model_creator, pydantic_queryset_creator
from .base import AbsModel, fields
class User(AbsModel):
username = fields.CharField(60)
age = fields.IntField()
class Meta:
table = "users"
def __str__(self):
return self.name
User_Pydantic = pydantic_model_creator(User)
UserIn_Pydantic = pydantic_model_creator(User, name="UserIn", exclude_readonly=True)
User_Pydantic_List = pydantic_queryset_creator(User)
models/__init__.py
from .users import User # NOQA: F401
tests/test_users.py
import pytest
from httpx import AsyncClient
from models.users import User
#pytest.mark.anyio
async def test_testpost(client: AsyncClient):
name, age = ["sam", 99]
assert await User.filter(username=name).count() == 0
data = {"username": name, "age": age}
response = await client.post("/testpost", json=data)
assert response.json() == dict(data, id=1)
assert response.status_code == 200
response = await client.get("/users")
assert response.status_code == 200
assert response.json() == [dict(data, id=1)]
assert await User.filter(username=name).count() == 1
Source code of the demo had been post to github:
https://github.com/waketzheng/fastapi-tortoise-pytest-demo.git

FastAPI variable query parameters

I am writing a Fast API server that accepts requests, checks if users are authorized and then redirects them to another URL if successful.
I need to carry over URL parameters, e.g. http://localhost:80/data/?param1=val1&param2=val2 should redirect to
http://some.other.api/?param1=val1&param2=val2, thus keeping previously allotted parameters.
The parameters are not controlled by me and could change at any moment.
How can I achieve this?
Code:
from fastapi import FastAPI
from starlette.responses import RedirectResponse
app = FastAPI()
#app.get("/data/")
async def api_data():
params = '' # I need this value
url = f'http://some.other.api/{params}'
response = RedirectResponse(url=url)
return response

In the docs they talk about using the Request directly, which then lead me to this:
from fastapi import FastAPI, Request
from starlette.responses import RedirectResponse
app = FastAPI()
#app.get("/data/")
async def api_data(request: Request):
params = request.query_params
url = f'http://some.other.api/?{params}'
response = RedirectResponse(url=url)
return response

If the query parameters are known when starting the API but you still wish to have them dynamically set:
from fastapi import FastAPI, Depends
from pydantic import create_model
app = FastAPI()
# Put your query arguments in this dict
query_params = {"name": (str, "me")}
query_model = create_model("Query", **query_params) # This is subclass of pydantic BaseModel
# Create a route
#app.get("/items")
async def get_items(params: query_model = Depends()):
params_as_dict = params.dict()
...
This has the benefit that you see the parameters in the automatic documentation:
But you are still able to define them dynamically (when starting the API).
Note: if your model has dicts, lists or other BaseModels as field types, the request body pops up. GET should not have body content so you might want to avoid those types.
See more about dynamic model creation from Pydantic documentation.

As mention in docs of FastAPI https://fastapi.tiangolo.com/tutorial/query-params-str-validations/.
#app.get("/")
def read_root(param1: Optional[str] = None, param2: Optional[str] = None):
url = f'http://some.other.api/{param1}/{param2}'
return {'url': str(url)}
output

I use a combination of Depends, BaseModel and the Request object itself.
Here's an example for a HTTP request like localhost:5000/api?requiredParam1=value1&optionalParam2=value2&dynamicParam1=value3&dynamicParam2=value4
# imports
from typing import Union
from pydantic import BaseModel
from fastapi import Depends, Request
# the base model
class QueryParams(BaseModel):
required: str
optional: Union[None, str] = None
dynamic: dict
# dependency
async def query_params(
request: Request, requiredParam1: str, optionalParam1: Union[None, str] = None
):
# process the request here
dynamicParams = {}
for k in request.query_params.keys():
if 'dynamicParam' not in k:
continue
dynamicParams[k] = request.query_params[k]
# also maybe do some other things on the arguments
# ...
return {
'required': requiredParam1,
'optional': optionalParam1,
'dynamic': dynamicParams
}
# the endpoint
#app.get("api/")
async def hello(params: QueryParams = Depends(query_params)):
# Maybe do domething with params here,
# Use it as you would any BaseModel object
# ...
return params
Refer the Starlette documentation on how to use the request object: https://www.starlette.io/requests/
Note that you can put query_params in a different module, and need not add any more code to explicitly pass the Request object. FastAPI already does that when you make a call to the endpoint :)

This is a code I derived from #Hajar Razip using a more pydantic like approach:
from pydantic import (
BaseModel,
)
from typing import (
Dict,
List,
Optional,
)
from fastapi import (
Depends,
FastAPI,
Query,
Request,
)
class QueryParameters(BaseModel):
"""Model for query parameter."""
fixId: Optional[str]
fixStr: Optional[str]
fixList: Optional[List[str]]
fixBool: Optional[bool]
dynFields: Dict
_aliases: Dict[str,str] = {"id": "fixId"}
#classmethod
def parser(
cls,
request: Request,
fixId: Optional[str] = Query(None, alias="id"),
fixStr: Optional[str] = Query(None),
fixList: Optional[List[str]] = Query(None),
fixBool: bool = Query(True),
) -> Dict:
"""Parse query string parameters."""
dynFields = {}
reserved_keys = cls.__fields__
query_keys = request.query_params
for key in query_keys:
key = cls._aliases.get(key, key)
if key in reserved_keys:
continue
dynFields[key] = request.query_params[key]
return {
"fixId": fixId,
"fixStr": fixStr,
"fixList": fixList,
"fixBool": fixBool,
"dynFields": dynFields
}
app = FastAPI()
#app.get("/msg")
def get_msg(
parameters: QueryParameters = Depends(
QueryParameters.parser,
),
) -> None:
return parameters
The output documentation is then
Here it is the result of calling GET /msg
> curl -s -X 'GET' 'http://127.0.0.1:8000/msg?id=Victor&fixStr=hi&fixList=eggs&fixList=milk&fixList=oranges&fixBool=true' -H 'accept: application/json' | python3 -m json.tool
{
"fixId": "Victor",
"fixStr": "hi",
"fixList": [
"eggs",
"milk",
"oranges"
],
"fixBool": true,
"dynFields": {}
}
Here it is the GET /msg call using dynamic fields
> curl -s -X 'GET' 'http://127.0.0.1:8000/msg?id=Victor&fixStr=hi&fixList=eggs&fixList=milk&fixList=oranges&fixBool=true&key1=value1&key2=value2' -H 'accept: application/json' | python3 -m json.tool
{
"fixId": "Victor",
"fixStr": "hi",
"fixList": [
"eggs",
"milk",
"oranges"
],
"fixBool": true,
"dynFields": {
"key1": "value1",
"key2": "value2"
}
}