I am trying to do get requests with proxy support however, it simply does not hit that part of the code.
proxy = f"http://{line}"
try:
async with aiohttp.ClientSession() as session:
async with session.get('https://www.google.com', proxy=proxy, timeout=1) as resp:
text = await resp.read()
print(f'Status: {text}')
if(resp.status == 200):
if(proxy not in working_proxies):
working_proxies.append(proxy)
print(f'{Fore.GREEN}[PROXY] {proxy} did work.')
sum += 1
except(Exception) as e:
print(f'Error: {e}')
It does not hit the await resp.text() and does not print it out at all.
Related
I am coding a bot that changes the channel name when a certain website is online. This is the code I've got and it works perfectly fine until I realised I had to await: channel.edit(name="website-is-up") For some reason the error catching just completely stopped working.
Here is my code:
async def timer():
await bot.wait_until_ready()
channel = bot.get_channel(123456) #actual channel id is in my code but I obfuscated it because it thought it might have been private
while True:
try:
response = requests.head(url)
except Exception as e:
print(f"NOT OK: {str(e)}")
channel.edit(name="website-is-down")
else:
if response.status_code == 200:
print("OK")
channel.edit(name="website-is-up")
else:
print(f"NOT OK: HTTP response code {response.status_code}")
channel.edit(name="website-is-down")
await asyncio.sleep(0)
bot.loop.create_task(timer())
Thanks in advance
As mentioned in the comments, requests blocks your code so you need to use aiohttp. See head and its return object ClientResponse.
async with aiohttp.ClientSession() as session:
r = session.head('example.com')
status_code = r.status
# (int) 200
I'm new to Async and, I wanna create a script that will execute a request every half second to check if a website is available. So even if the website response time is like '4s' it will execute another request every 0.5 seconds. Once one of the requests receives the "200" status code the event loop will break.
URL = "https://stackoverflow.com"
async def load(session, url):
async with session.get(url) as response:
return await response.status == 200
async def create_session():
complete = False
while not complete:
async with aiohttp.ClientSession() as session:
task = await asyncio.create_task(load(session, URL))
if task:
break
await asyncio.sleep(0.5)
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(create_session())
Right now I got something like this which obviously won't work.
I was able to create the desired program using the asyncio.Event().
import asyncio
import aiohttp
url = "https://www.somesite.com"
async def load(session, url, flag):
async with session.get(url) as response:
if await response.status == 200: #Check if the site is available.
flag.set() # Flag is set
async def create_session():
flag = asyncio.Event()
async with aiohttp.ClientSession() as session: # Create aiohttp Session
while 1:
asyncio.create_task(load(session, url, flag))
await asyncio.sleep(0.5) # Wait 0.5 s between the requests
if flag.is_set():# If flag is set then break the loop
break
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(create_session())
I need to send over 1 million HTTP requests and so far every option I've tried is just way too slow. I thought I could speed it up with aiohttp but that doesn't seem any faster than requests.
I was trying to do it with python but I'm open to other options as well.
Here is the code using both requests and aiohttp, any tips for speeding up the process?
requests code:
import requests
url = 'https://mysite.mysite:443/login'
users = [line.strip() for line in open("ids.txt", "r")]
try:
for user in users:
r = requests.post(url,data ={'username':user})
if 'login.error.invalid.username' not in r.text:
print(user, " is valid")
else:
print(user, " not found")
except Exception as e:
print(e)
aiohttp code:
import aiohttp
import asyncio
url = 'https://mysite.mysite:443/login'
users = [line.strip() for line in open("ids.txt", "r")]
async def main():
async with aiohttp.ClientSession() as session:
try:
for user in users:
payload = {"timeZoneOffSet": "240", "useragent": '', "username": user}
async with session.post(url, data=payload) as resp:
if 'login.error.invalid.username' not in await resp.text():
print(user, " is valid")
else:
print(user, " not found")
except Exception as e:
print(e)
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
You could use an asyncio.gather to collect results from a bunch of requests working in parallel.
Warning: code is just an example and is not tested.
import asyncio
from aiohttp import ClientSession
async def fetch(url, session, payload):
async with session.post(url, data=payload) as resp:
if 'login.error.invalid.username' not in await resp.text():
print(user, " is valid")
else:
print(user, " not found")
async def run(r):
url = "http://your_url:8000/{}"
tasks = []
async with ClientSession() as session:
for i in range(r):
task = asyncio.ensure_future(fetch(url.format(i), session))
tasks.append(task)
responses = await asyncio.gather(*tasks)
# you now have all response bodies in this variable
def print_responses(result):
print(result)
loop = asyncio.get_event_loop()
future = asyncio.ensure_future(run(4))
loop.run_until_complete(future)
I have the following code
import asyncio
import aiohttp
urls = [
'http://54.224.27.241',
'http://54.224.27.241',
'http://54.224.27.241',
'http://54.224.27.241',
'http://54.224.27.241',
]
async def query(urls):
out = []
with aiohttp.ClientSession() as session:
for url in urls:
try:
async with session.get(url, timeout=5) as resp:
text = await resp.text()
out.append(resp.status)
except:
print('timeout')
return out
loop = asyncio.get_event_loop()
out = loop.run_until_complete(query(urls))
loop.close()
print(str(out))
The code is much slower than the one that uses a threadpool and keep increasing if you increase the number of urls (lets say 20, 50 etc.)
I have a feeling that when the initial connection establishment is not done in an async way.
(Note that I am connecting here to an non-existing server to deliberately produce a connection timeout).
Can someone point out what is wrong here?
Warning: I don't promise this code works, as I can't install aiohttp atm, but looking at the example in the docs
async def fetch(session, url):
async with async_timeout.timeout(10):
async with session.get(url) as response:
return await response.text()
async def main():
async with aiohttp.ClientSession() as session:
html = await fetch(session, 'http://python.org')
print(html)
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
Notice how they're calling the aiohttp.ClientSession() with the async keyword. Additionally, I was getting some error in your line data = await async with session.get(url) as resp:
async def fetch(session, url):
async with session.get(url) as response:
return await response.text()
async def main():
out = []
async with aiohttp.ClientSession() as session:
for url in urls:
data = await fetch(session, url)
out.append(data)
return out
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
I am trying to teach myself Python's async functionality. To do so I have built an async web scraper. I would like to limit the total number of connections I have open at once to be a good citizen on servers. I know that semaphore's are a good solution, and the asyncio library has a semaphore class built in. My issue is that Python complains when using yield from in an async function as you are combining yield and await syntax. Below is the exact syntax I am using...
import asyncio
import aiohttp
sema = asyncio.BoundedSemaphore(5)
async def get_page_text(url):
with (yield from sema):
try:
resp = await aiohttp.request('GET', url)
if resp.status == 200:
ret_val = await resp.text()
except:
raise ValueError
finally:
await resp.release()
return ret_val
Raising this Exception:
File "<ipython-input-3-9b9bdb963407>", line 14
with (yield from sema):
^
SyntaxError: 'yield from' inside async function
Some possible solution I can think of...
Just use the #asyncio.coroutine decorator
Use threading.Semaphore? This seems like it may cause other issues
Try this in the beta of Python 3.6 for this reason.
I am very new to Python's async functionality so I could be missing something obvious.
You can use the async with statement to get an asynchronous context manager:
#!/usr/local/bin/python3.5
import asyncio
from aiohttp import ClientSession
sema = asyncio.BoundedSemaphore(5)
async def hello(url):
async with ClientSession() as session:
async with sema, session.get(url) as response:
response = await response.read()
print(response)
loop = asyncio.get_event_loop()
loop.run_until_complete(hello("http://httpbin.org/headers"))
Example taken from here. The page is also a good primer for asyncio and aiohttp in general.
OK, so this is really silly but I just replaces yield from with await in the semaphore context manager and it is working perfectly.
sema = asyncio.BoundedSemaphore(5)
async def get_page_text(url):
with (await sema):
try:
resp = await aiohttp.request('GET', url)
if resp.status == 200:
ret_val = await resp.text()
except:
raise ValueError
finally:
await resp.release()
return ret_val
For the semaphore only:
sem = asyncio.Semaphore(10)
# ... later
async with sem:
# work with shared resource
which is equivalent to:
sem = asyncio.Semaphore(10)
# ... later
await sem.acquire()
try:
# work with shared resource
finally:
sem.release()
ref:
https://docs.python.org/3/library/asyncio-sync.html#asyncio.Semaphore