I try to reuse HTTP-session as aiohttp docs advice
Don’t create a session per request. Most likely you need a session per
application which performs all requests altogether.
But usual pattern which I use with requests lib doesn`t work:
def __init__(self):
self.session = aiohttp.ClientSession()
async def get_u(self, id):
async with self.session.get('url') as resp:
json_resp = await resp.json()
return json_resp.get('data', {})
Then I try to
await client.get_u(1)
I got error
RuntimeError: Timeout context manager should be used inside a task
Any workarounds with async_timeout didn't help.
Another way is working:
async def get_u(self, id):
async with aiohttp.ClientSession() as session:
with async_timeout.timeout(3):
async with session.get('url') as resp:
json_resp = await resp.json()
return json_resp.get('data', {})
But it seems like creating session per request.
So my question: how to properly reuse aiohttp-session?
UPD: minimal working example. Sanic application with following view
import aiohttp
from sanic.views import HTTPMethodView
class Client:
def __init__(self):
self.session = aiohttp.ClientSession()
self.url = 'https://jsonplaceholder.typicode.com/todos/1'
async def get(self):
async with self.session.get(self.url) as resp:
json_resp = await resp.json()
return json_resp
client = Client()
class ExView(HTTPMethodView):
async def get(self, request):
todo = await client.get()
print(todo)
I had the same error. The solution for me was initializing the client within an async function. EG:
class SearchClient(object):
def __init__(self, search_url: str, api_key: str):
self.search_url = search_url
self.api_key = api_key
self.session = None
async def _get(self, url, attempt=1):
if self.session is None:
self.session = aiohttp.ClientSession(raise_for_status=True)
headers = {
'Content-Type': 'application/json',
'api-key': self.api_key
}
logger.info("Running Search: {}".format(url))
try:
with timeout(60):
async with self.session.get(url, headers=headers) as response:
results = await response.json()
return results
For example you can create ClientSession on app start (using on_startup signal https://docs.aiohttp.org/en/stable/web_advanced.html#signals).
Store it to you app (aiohttp application has dict interface for such issues https://aiohttp.readthedocs.io/en/stable/faq.html#id4) and get access to your session through request.app['YOU_CLIENT_SESSION'] in request.
Related
I have a pretty complicated API with custom parameters and headers so I created a class to wrap around it. Here's a contrived example:
import asyncio
import aiohttp
# The wrapper class around my API
class MyAPI:
def __init__(self, base_url: str):
self.base_url = base_url
async def send(self, session, method, url) -> aiohttp.ClientResponse:
request_method = getattr(session, method.lower())
full_url = f"{self.base_url}/{url}"
async with request_method(full_url) as response:
return response
async def main():
api = MyAPI("https://httpbin.org")
async with aiohttp.ClientSession() as session:
response = await api.send(session, "GET", "/uuid")
print(response.status) # 200 OK
print(await response.text()) # Exception: Connection closed
asyncio.run(main())
Why is my session closed? I didn't exit the context manager of session.
If I ignore the wrapper class, everything works as expected:
async def main():
async with aiohttp.ClientSession() as session:
async with session.get("https://httpbin.org/uuid") as response:
print(await response.text())
You can't call response.text() once you have left the request_method(full_url) context.
If you write:
async with request_method(full_url) as response:
text = await response.text()
return response.status, text
then the send() method returns without error.
I have the static url, headers, and data.
Is it possible to make million post requests simultaneously with python?
This is the file.py:
import json
import requests
url = "https://abcd.com"
headers = "headers"
body = "body"
resp = requests.post(url, headers=headers, data=body)
json_resp = json.loads(resp.content)["data"]
print(json_resp)
You might want to use some python tools for that such as:
https://locust.io/
Your file would look like:
from locust import HttpUser, task, between
class QuickstartUser(HttpUser):
#task
def task_name(self):
self.client.post(url, headers=headers, data=body)
You could feed it to locust in such a way:
locust --headless --users <number_of_user> -f <your_file.py>
You can do this in several ways, which is the best method and idea of async work
The second method is ThreadPoolExecutor, which I do not highly recommend
there's a example for do this.
# modified fetch function with semaphore
import random
import asyncio
from aiohttp import ClientSession
async def fetch(url, session):
async with session.get(url) as response:
delay = response.headers.get("DELAY")
date = response.headers.get("DATE")
print("{}:{} with delay {}".format(date, response.url, delay))
return await response.read()
async def bound_fetch(sem, url, session):
# Getter function with semaphore.
async with sem:
await fetch(url, session)
async def run(r):
url = "http://localhost:8080/{}"
tasks = []
# create instance of Semaphore
sem = asyncio.Semaphore(1000)
# Create client session that will ensure we dont open new connection
# per each request.
async with ClientSession() as session:
for i in range(r):
# pass Semaphore and session to every GET request
task = asyncio.ensure_future(bound_fetch(sem, url.format(i), session))
tasks.append(task)
responses = asyncio.gather(*tasks)
await responses
number = 10000
loop = asyncio.get_event_loop()
future = asyncio.ensure_future(run(number))
loop.run_until_complete(future)
I am building own async python package and faced the problem.
This is my code:
class Client:
"""
Async client for making requests
"""
def __init__(self, base_url: str = BASE_URL) -> None:
self.base_url = base_url
self.session = ClientSession()
async def get(self, method: str, *args: tp.Any, **kwargs: tp.Any) -> tp.Any:
async with self.session.get(f'{self.base_url}/{method}', *args, **kwargs) as response:
data = await response.json()
return data
When I try to use something like this:
await client.get()
I get
RuntimeError: Timeout context manager should be used inside a task
I suppose that the reason of this error is calling ClientSession() not inside the coroutine. But I hope that somebody knows the way to re-use ClientSession()
I have already read other similar questions, but they are not suitable to my situation.
You can initialize (and cache) the session when needed:
class Client:
"""
Async client for making requests
"""
def __init__(self, base_url: str = BASE_URL) -> None:
self.base_url = base_url
self.session = None
async def get(self, method: str, *args: tp.Any, **kwargs: tp.Any) -> tp.Any:
if not self.session:
self.session = ClientSession()
async with self.session.get(f'{self.base_url}/{method}', *args, **kwargs) as response:
data = await response.json()
return data
Depending on how you use the Client you can also use a class attribute for the session object.
Update:
ClientSession creation should be protected from race condition using Mutex:
_session_mutex = asyncio.Lock()
async def __create_session_if_required(self):
if self.session is None:
async with self._session_mutex:
if self.session is None:
self.session = aiohttp.ClientSession()
# should be closed if not Singleton class: "await session.close()"
async def get(..):
await self.__create_session_if_required()
async with self.session.get() as response:
# ...
I'm writing a class that will do http requests using aiohttp. According to the docs I should not to create a ClientSession per request, so I want to reuse the same session.
code:
class TestApi:
def __init__(self):
self.session = aiohttp.ClientSession()
# async defs methods from here
When doing
TestApi()
I get the error: Unclosed client session.
What is the solution to persist the ClientSession object?
The expression TestApi() on a line by itself creates a TestApi object and immediately throws it away. aiohttp complaints that the session was never closed (either by leaving an async with block or with an explicit call to close()), but even without the warning it doesn't make sense not to assign the API object to a variable where it will be actually used.
To reuse the session, your code needs to have access to the session, or to an object that holds it:
async def fetch(url):
async with aiohttp.request('GET', url) as resp:
resp.raise_for_status()
return await resp.read()
async def main():
url1_data, url2_data = asyncio.gather(
fetch('http://url1'), fetch('http://url2'))
url3_data, url4_data = asyncio.gather(
fetch('http://url3'), fetch('http://url4'))
One option is to add a session parameter to fetch (and other functions) and consistently call it with a session created in main(). A better option is to create an API class and convert the global functions like fetch to methods:
class Http:
async def __aenter__(self):
self._session = aiohttp.ClientSession()
return self
async def __aexit__(self, *err):
await self._session.close()
self._session = None
async def fetch(self, url):
async with self._session.get(url) as resp:
resp.raise_for_status()
return await resp.read()
main() can still exist as a function, but it can consistently use the object that holds the session:
async def main():
async with Http() as http:
url1_data, url2_data = await asyncio.gather(
http.fetch('http://url1'), http.fetch('http://url2'))
url3_data, url4_data = await asyncio.gather(
http.fetch('http://url3'), http.fetch('http://url4'))
In the above code, the async with statement is used to ensure that the session is closed whenever the scope is left.
My code is as follows:
import asyncio
import aiohttp
urls = [
'http://www.163.com/',
'http://www.sina.com.cn/',
'https://www.hupu.com/',
'http://www.csdn.net/'
]
async def get_url_data(u):
"""
read url data
:param u:
:return:
"""
print('running ', u)
resp = await aiohttp.ClientSession().get(url=u)
headers = resp.headers
print(u, headers)
return headers
async def request_url(u):
"""
main func
:param u:
:return:
"""
res = await get_url_data(u)
return res
loop = asyncio.get_event_loop()
task_lists = asyncio.wait([request_url(u) for u in urls])
loop.run_until_complete(task_lists)
loop.close()
When i running my code, it's display a warning message:
Unclosed client session
Anybody can give me some solutions about that?
Thanks a lot
You should close the connection in the end.
You have 2 options:
You can close the connection manually:
import aiohttp
session = aiohttp.ClientSession()
# use the session here
session.close()
Or you can use it with a contex manager:
import aiohttp
import asyncio
async def fetch(client):
async with client.get('http://python.org') as resp:
assert resp.status == 200
return await resp.text()
async def main(loop):
async with aiohttp.ClientSession(loop=loop) as client:
html = await fetch(client)
print(html)
loop = asyncio.get_event_loop()
loop.run_until_complete(main(loop))
The client session supports the context manager protocol for self closing.
If you are not using context manager, the proper way to close it would also need an await. Many answers on the internet miss that part, and few people actually notice it, presumably because most people use the more convenient context manager. But the manual await session.close() is essential when/if you are closing a class-wide session inside the tearDownClass() when doing unittesting.
import aiohttp
session = aiohttp.ClientSession()
# use the session here
await session.close()
You should use ClientSession using async context manager for proper blocking/freeing resources:
async def get_url_data(u):
"""
read url data
:param u:
:return:
"""
print('running ', u)
async with aiohttp.ClientSession() as session:
resp = await session.get(url=u)
headers = resp.headers
print(u, headers)
return headers