How can I handling PascalCase and camelCase requests bodies to snake_case same time in FastAPI app?
I tried to use middleware and routerHandler to replace camelCase to PascalCase, but it works not so good.
class CustomRouteHandler(APIRoute):
def get_route_handler(self) -> Callable:
original_route_handler = super().get_route_handler()
async def custom_route_handler(request: Request) -> Response:
route_path = request.url.path
body = await request.body()
logger.info({"path": request.url.path, "request": request._body.decode("utf-8")})
if body:
body = ujson.dumps(humps.pascalize(ujson.loads(body.decode("utf-8")))).encode("ascii")
request._body = body
try:
return await original_route_handler(request)
except ValidationError as e:
logger.exception(e, exc_info=True)
return UJSONResponse(status_code=200, content={"Success": False, "Message": e})
return custom_route_handler
router = APIRouter(prefix="/payments", route_class=CustomRouteHandler)
When I logging this code, all fine. But it returns ValidationError:
request body: {"test": 12345}
logger after pascalize: {"Test": 12345}
ERROR: 1 validation error for Request\nbody -> Test
none is not an allowed value (type=type_error.none.not_allowed)
First of all you should understand that when we are using CamelCase in API requests, the request body will be converted into a string by default. So, if you want to handle both camelCase and PascalCase, you need to handle both cases. The easiest way is to add two routes - one for each case. For example:
class CustomRouteHandler(APIRoute):
def get_route_handler(self) -> Callable: original_route_handler = super().get_route_handler()
async def custom_route_handler(request: Request) -> Response: route_path = request.url.path body = await request.body()
logger.info(\"Path\": request.url.path, \"Request\": request._body.decode("utf-8"))
if body: body = ujson.dumps(humps.pascalize(ujson.loads(body.decode("utf-8")))).encode("ascii")
request._body = body
try: return await original_route_handler(request)
except ValidationError as e: logger.exception(e, exc_info=True)
return UJSONResponse(status_code=200, content=\"Success\", "Message": e)
return custom_route_handler router = APIRouter(prefix="/payments", route_class=CustomRouteHandler)
Then you can create a middleware which handles both cases:
middleware = RouterMiddleware(prefix="/payments", route_class="custom_route_handler")
And then you can use your custom routing handler in your main route:
main_routes = Routes.new(Router())
You can see more examples on the official docs page.
Related
I have a pytest function to test a FastAPI endpoint, this is just to verify that right data can create an entity, here the test function:
import pytest
from databases import Database
from fastapi import FastAPI, status
from httpx import AsyncClient
from modules.member.members_schemas import MemberCreate
pytestmark = pytest.mark.asyncio
class TestCreateMember:
async def test valid_input_creates_member(self, app: FastAPI, client: AsyncClient,
db: Database) -> None:
new_member = MemberCreate(
fullname="John Smith"
dni="123456789",
birthdate=datetime(1985, 10, 4),
email="testemail#test.com",
)
res = await client.post(
app.url_path_for("members:create-member), json={"member": new_member.dict()}
)
assert res.status_code == status.HTTP_201_CREATED
Here the pydantic model (schema in my case):
from datetime import datetime
from pydantic import BaseModel, BaseConfig, EmailStr
class BaseSchema(BaseModel):
class Config(BaseConfig):
allow_population_by_field_name = True
orm_mode = True
class MemberCreate(BaseSchema):
fullname: str
dni: str
birth_date: datetime
email: EmailStr
And here is the endpoint to test:
from databases import Database
from fastapi import APIRouter, Body, Depends, status
router = APIRouter(
prefix="/members",
tags=["members"],
responses={404: {"description": "Not found"}},
)
#router.post(
"/",
response_model=MemberPublic,
name="members:create-member",
status_code=status.HTTP_201_CREATED,
)
async def create_member(
member: MemberCreate = Body(..., embed=True),
db: Database = Depends(get_database),
current_user: UserInDB = Depends(get_current_active_user),
) -> ServiceResult:
result = await MemberService(db).create_member(member, current_user)
return handle_result(result)
ServiceResult type and handle_result() function are funcionalies to make a stardard answer from each endpoint made it, not problem working with it with a lot of other endpoints. When I ran pytest on this particular test, I got this error:
self = <json.encoder.JSONEncoder object at 0x7fe1ebe3fd10>
o = datetime.datetime(1985, 10, 4, 0, 0)
def default(self, o):
"""Implement this method in a subclass such that it returns
a serializable object for ``o``, or calls the base implementation
(to raise a ``TypeError``).
For example, to support arbitrary iterators, you could
implement default like this::
def default(self, o):
try:
iterable = iter(o)
except TypeError:
pass
else:
return list(iterable)
# Let the base class default method raise the TypeError
return JSONEncoder.default(self, o)
"""
> raise TypeError(f'Object of type {o.__class__.__name__} '
f'is not JSON serializable')
E TypeError: Object of type datetime is not JSON serializable
So I changed the test adding a json.dumps() (importing json, of course), this way:
async def test valid_input_creates_member(self, app: FastAPI, client: AsyncClient,
db: Database) -> None:
new_member = MemberCreate(
fullname="John Smith"
dni="123456789",
birthdate=datetime(1985, 10, 4),
email="testemail#test.com",
)
new_member_json = json.dumps(new_member.dict(), indent=4, sort_keys=True, default=str)
res = await client.post(
app.url_path_for("members:create-member), json={"member": new_member_json}
)
assert res.status_code == status.HTTP_201_CREATED
After I ran pytest again, I got this error new error:
> assert res.status_code == status.HTTP_201_CREATED
E assert 422 == 201
E + where 422 = <Response [422 Unprocessable Entity]>.status_code
E + and 201 = status.HTTP_201_CREATED
Which is a Pydantic validation error at the endpoint.
When I see the endpoint post in the swagger view, the datetime field is presented as a string, so no problems to be serialized, but I don´t want to change the schema to receive a str, because I would lose the pydantic's power. So my question is, How can I set the test in order to override the pydantc validation and accept the datetime field? or I must change the schema to have a string field and process it internally in order to receive only valid datetime data?. I'll appreciate any help, because I'm stuck on this problem for several hours.
As suggets matslindh, I changed the test function, eliminating the pydantic object and it was substituted by a simple dict, this way:
class TestCreateMember:
async def test valid_input_creates_member(self, app: FastAPI, client: AsyncClient,
db: Database) -> None:
new_member = {
"fullname": "John Smith"
"dni": "123456789",
"birthdate": "1985-10-04T00:00",
"email": "testemail#test.com",
}
res = await client.post(
app.url_path_for("members:create-member), json={"member": new_member}
)
assert res.status_code == status.HTTP_201_CREATED
After that everything works as expected
In the following example when you pass a username in the basic auth field it raise a basic 400 error, but i want to return 401 since it's related to the authentication system.
I did tried Fastapi exceptions classes but they do not raise (i presume since we are in a starlette middleware). Il also tried JSONResponse from starlette but it doesn't work either.
AuthenticationError work and raise a 400 but it's juste an empty class that inherit from Exception so no status code can be given.
Fully working example:
import base64
import binascii
import uvicorn
from fastapi import Depends, FastAPI, HTTPException, Request
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer, HTTPBasic
from starlette.authentication import AuthenticationBackend, AuthCredentials, AuthenticationError, BaseUser
from starlette.middleware import Middleware
from starlette.middleware.authentication import AuthenticationMiddleware
from starlette.responses import JSONResponse
class SimpleUserTest(BaseUser):
"""
user object returned to route
"""
def __init__(self, username: str, test1: str, test2: str) -> None:
self.username = username
self.test1 = test1
self.test2 = test2
#property
def is_authenticated(self) -> bool:
return True
async def jwt_auth(auth: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False))):
if auth:
return True
async def key_auth(apikey_header=Depends(HTTPBasic(auto_error=False))):
if apikey_header:
return True
class BasicAuthBackend(AuthenticationBackend):
async def authenticate(self, conn):
if "Authorization" not in conn.headers:
return
auth = conn.headers["Authorization"]
try:
scheme, credentials = auth.split()
if scheme.lower() == 'bearer':
# check bearer content and decode it
user: dict = {"username": "bearer", "test1": "test1", "test2": "test2"}
elif scheme.lower() == 'basic':
decoded = base64.b64decode(credentials).decode("ascii")
username, _, password = decoded.partition(":")
if username:
# check redis here instead of return dict
print("try error raise")
raise AuthenticationError('Invalid basic auth credentials') # <= raise 400, we need 401
# user: dict = {"username": "basic auth", "test1": "test1", "test2": "test2"}
else:
print("error should raise")
return JSONResponse(status_code=401, content={'reason': str("You need to provide a username")})
else:
return JSONResponse(status_code=401, content={'reason': str("Authentication type is not supported")})
except (ValueError, UnicodeDecodeError, binascii.Error) as exc:
raise AuthenticationError('Invalid basic auth credentials')
return AuthCredentials(["authenticated"]), SimpleUserTest(**user)
async def jwt_or_key_auth(jwt_result=Depends(jwt_auth), key_result=Depends(key_auth)):
if not (key_result or jwt_result):
raise HTTPException(status_code=401, detail="Not authenticated")
app = FastAPI(
dependencies=[Depends(jwt_or_key_auth)],
middleware=[Middleware(AuthenticationMiddleware, backend=BasicAuthBackend())]
)
#app.get("/")
async def read_items(request: Request) -> str:
return request.user.__dict__
if __name__ == "__main__":
uvicorn.run("main:app", host="127.0.0.1", port=5000, log_level="info")
if we set username in basic auth:
INFO: 127.0.0.1:22930 - "GET / HTTP/1.1" 400 Bad Request
so i ended up using on_error as suggested by #MatsLindh
old app:
app = FastAPI(
dependencies=[Depends(jwt_or_key_auth)],
middleware=[
Middleware(
AuthenticationMiddleware,
backend=BasicAuthBackend(),
)
],
)
new version:
app = FastAPI(
dependencies=[Depends(jwt_or_key_auth)],
middleware=[
Middleware(
AuthenticationMiddleware,
backend=BasicAuthBackend(),
on_error=lambda conn, exc: JSONResponse({"detail": str(exc)}, status_code=401),
)
],
)
I choose to use JSONResponse and return a "detail" key/value to emulate a classic 401 fastapi httperror
So, I have created a Custom Middleware for my big FastAPI Application, which alters responses from all of my endpoints this way:
Response model is different for all APIs. However, my MDW adds meta data to all of these responses, in an uniform manner. This is what the final response object looks like:
{
"data": <ANY RESPONSE MODEL THAT ALL THOSE ENDPOINTS ARE SENDING>,
"meta_data":
{
"meta_data_1": "meta_value_1",
"meta_data_2": "meta_value_2",
"meta_data_3": "meta_value_3",
}
}
So essentially, all original responses, are wrapped inside a data field, a new field of meta_data is added with all meta_data. This meta_data model is uniform, it will always be of this type:
"meta_data":
{
"meta_data_1": "meta_value_1",
"meta_data_2": "meta_value_2",
"meta_data_3": "meta_value_3",
}
Now the problem is, when the swagger loads up, it shows the original response model in schema and not the final response model which has been prepared. How to alter swagger to reflect this correctly?
I have tried this:
# This model is common to all endpoints!
# Since we are going to add this for all responses
class MetaDataModel(BaseModel):
meta_data_1: str
meta_data_2: str
meta_data_3: str
class FinalResponseForEndPoint1(BaseModel):
data: OriginalResponseForEndpoint1
meta_data: MetaDataModel
class FinalResponseForEndPoint2(BaseModel):
data: OriginalResponseForEndpoint2
meta_data: MetaDataModel
and so on ...
This approach does render the Swagger perfectly, but there are 2 major problems associated with it:
All my FastAPI endpoints break and give me an error when they are returning response. For example: my endpoint1 is still returning the original response but the endpoint1 expects it to send response adhering to FinalResponseForEndPoint1 model
Doing this approach for all models for all my endpoints, does not seem like the right way
Here is a minimal reproducible example with my custom middleware:
from starlette.types import ASGIApp, Receive, Scope, Send, Message
from starlette.requests import Request
import json
from starlette.datastructures import MutableHeaders
from fastapi import FastAPI
class MetaDataAdderMiddleware:
application_generic_urls = ['/openapi.json', '/docs', '/docs/oauth2-redirect', '/redoc']
def __init__(
self,
app: ASGIApp
) -> None:
self.app = app
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if scope["type"] == "http" and not any([scope["path"].startswith(endpoint) for endpoint in MetaDataAdderMiddleware.application_generic_urls]):
responder = MetaDataAdderMiddlewareResponder(self.app, self.standard_meta_data, self.additional_custom_information)
await responder(scope, receive, send)
return
await self.app(scope, receive, send)
class MetaDataAdderMiddlewareResponder:
def __init__(
self,
app: ASGIApp,
) -> None:
"""
"""
self.app = app
self.initial_message: Message = {}
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
self.send = send
await self.app(scope, receive, self.send_with_meta_response)
async def send_with_meta_response(self, message: Message):
message_type = message["type"]
if message_type == "http.response.start":
# Don't send the initial message until we've determined how to
# modify the outgoing headers correctly.
self.initial_message = message
elif message_type == "http.response.body":
response_body = json.loads(message["body"].decode())
data = {}
data["data"] = response_body
data['metadata'] = {
'field_1': 'value_1',
'field_2': 'value_2'
}
data_to_be_sent_to_user = json.dumps(data, default=str).encode("utf-8")
headers = MutableHeaders(raw=self.initial_message["headers"])
headers["Content-Length"] = str(len(data_to_be_sent_to_user))
message["body"] = data_to_be_sent_to_user
await self.send(self.initial_message)
await self.send(message)
app = FastAPI(
title="MY DUMMY APP",
)
app.add_middleware(MetaDataAdderMiddleware)
#app.get("/")
async def root():
return {"message": "Hello World"}
If you add default values to the additional fields you can have the middleware update those fields as opposed to creating them.
SO:
from ast import Str
from starlette.types import ASGIApp, Receive, Scope, Send, Message
from starlette.requests import Request
import json
from starlette.datastructures import MutableHeaders
from fastapi import FastAPI
from pydantic import BaseModel, Field
# This model is common to all endpoints!
# Since we are going to add this for all responses
class MetaDataModel(BaseModel):
meta_data_1: str
meta_data_2: str
meta_data_3: str
class ResponseForEndPoint1(BaseModel):
data: str
meta_data: MetaDataModel | None = Field(None, nullable=True)
class MetaDataAdderMiddleware:
application_generic_urls = ['/openapi.json',
'/docs', '/docs/oauth2-redirect', '/redoc']
def __init__(
self,
app: ASGIApp
) -> None:
self.app = app
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if scope["type"] == "http" and not any([scope["path"].startswith(endpoint) for endpoint in MetaDataAdderMiddleware.application_generic_urls]):
responder = MetaDataAdderMiddlewareResponder(
self.app)
await responder(scope, receive, send)
return
await self.app(scope, receive, send)
class MetaDataAdderMiddlewareResponder:
def __init__(
self,
app: ASGIApp,
) -> None:
"""
"""
self.app = app
self.initial_message: Message = {}
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
self.send = send
await self.app(scope, receive, self.send_with_meta_response)
async def send_with_meta_response(self, message: Message):
message_type = message["type"]
if message_type == "http.response.start":
# Don't send the initial message until we've determined how to
# modify the outgoing headers correctly.
self.initial_message = message
elif message_type == "http.response.body":
response_body = json.loads(message["body"].decode())
response_body['meta_data'] = {
'field_1': 'value_1',
'field_2': 'value_2'
}
data_to_be_sent_to_user = json.dumps(
response_body, default=str).encode("utf-8")
headers = MutableHeaders(raw=self.initial_message["headers"])
headers["Content-Length"] = str(len(data_to_be_sent_to_user))
message["body"] = data_to_be_sent_to_user
await self.send(self.initial_message)
await self.send(message)
app = FastAPI(
title="MY DUMMY APP",
)
app.add_middleware(MetaDataAdderMiddleware)
#app.get("/", response_model=ResponseForEndPoint1)
async def root():
return ResponseForEndPoint1(data='hello world')
I don't think this is a good solution - but it doesn't throw errors and it does show the correct output in swagger.
In general I'm struggling to find a good way to document the changes/ additional responses that middleware can introduce in openAI/swagger. If you've found anything else I'd be keen to hear it!
Here is a fastapi webapp. I got some trouble in test.
post in crud.py:
async def post(payload: BookSchema):
query = books.insert().values(book=payload.book, author=payload.author,
create_time=payload.create_time, update_time=payload.update_time)
return await database.execute(query=query)
create_book:
#router.post("/book", response_model=BookDB, status_code=201)
async def create_book(payload: BookSchema):
book_id = await crud.post(payload)
response_object = {
"id": book_id,
**payload.dict()
}
return response_object
conftest.py:
import pytest
from starlette.testclient import TestClient
from app.main import app
#pytest.fixture()
def test_app():
with TestClient(app) as client:
yield client
I use pytest to test my case:
def test_create_book_invalid_json(test_app):
response = test_app.post(API_PREFIX + "/book", data=json.dumps({"book": "smart"}))
print(f"response.status_code is {response.status_code}")
assert response.status_code == 422
Expected status_code should be 422 and I got 201. I try to print response.status_code, and the output is 422.
test_books.py::test_create_book_invalid_json FAILED [100%]response.status_code is 422
test_books.py:20 (test_create_book_invalid_json)
201 != 422
Expected :422
Actual :201
You have not provided information about your test client here, but I assume it is a requests-like one. In such case the data parameter expects a dictionary, not a string.
response = test_app.post(url, data={'book': 'smart'})
This will send corresponding form data to your web app. If what you need is sending JSON, use the json parameter instead:
response = test_app.post(url, json={'book': 'smart'})
Hi I am experiencing weird behavior from SimpleHttpOperator.
I have extended this operator like this:
class EPOHttpOperator(SimpleHttpOperator):
"""
Operator for retrieving data from EPO API, performs token validity check,
gets a new one, if old one close to not valid.
"""
#apply_defaults
def __init__(self, entity_code, *args, **kwargs):
super().__init__(*args, **kwargs)
self.entity_code = entity_code
self.endpoint = self.endpoint + self.entity_code
def execute(self, context):
try:
token_data = json.loads(Variable.get(key="access_token_data", deserialize_json=False))
if (datetime.now() - datetime.strptime(token_data["created_at"],
'%Y-%m-%d %H:%M:%S.%f')).seconds >= 19 * 60:
Variable.set(value=json.dumps(get_EPO_access_token(), default=str), key="access_token_data")
self.headers = {
"Authorization": f"Bearer {token_data['token']}",
"Accept": "application/json"
}
super(EPOHttpOperator, self).execute(context)
except HTTPError as http_err:
logging.error(f'HTTP error occurred during getting EPO data: {http_err}')
raise http_err
except Exception as e:
logging.error(e)
raise e
And I have written a simple unit test:
def test_get_EPO_data(requests_mock):
requests_mock.get('http://ops.epo.org/rest-services/published-data/publication/epodoc/EP1522668',
text='{"text": "test"}')
requests_mock.post('https://ops.epo.org/3.2/auth/accesstoken',
text='{"access_token":"test", "status": "we just testing"}')
dag = DAG(dag_id='test_data', start_date=datetime.now())
task = EPOHttpOperator(
xcom_push=True,
do_xcom_push=True,
http_conn_id='http_EPO',
endpoint='published-data/publication/epodoc/',
entity_code='EP1522668',
method='GET',
task_id='get_data_task',
dag=dag,
)
ti = TaskInstance(task=task, execution_date=datetime.now(), )
task.execute(ti.get_template_context())
assert ti.xcom_pull(task_ids='get_data_task') == {"text": "test"}
Test doesn't pass though, the XCOM value from HttpHook is never pushed as an XCOM, I have checked that code responsible for the push logic in the hook class gets called:
....
if self.response_check:
if not self.response_check(response):
raise AirflowException("Response check returned False.")
if self.xcom_push_flag:
return response.text
What did I do wrong? Is this a bug?
So I actually managed to make it work by setting an xcom value to the result of super(EPOHttpOperator, self).execute(context).
def execute(self, context):
try:
.
.
.
self.headers = {
"Authorization": f"Bearer {token_data['token']}",
"Accept": "application/json"
}
super(EPOHttpOperator, self).execute(context) -> Variable.set(value=super(EPOHttpOperator, self).execute(context),key='foo')
Documentation is kind of misleading on this one; or am I doing something wrong after all?