We Keep Coding

How to use multiple proxies with requests library, python?

I have a list of proxies which I want requests lib to use it. Because some of them don't work I want to change the proxy each time one doesn't respond.
I have this code(I just tried if it would work this way it's not final)
import requests
import itertools
HTTP = [List of HTTP proxies...]
HTTPS = [list of https proxies...]
def try_proxies(http_proxies, https_proxies):
for proxy_http_element in http_proxies:
http_proxy = proxy_http_element
yield http_proxy
for proxy_https_element in https_proxies:
https_proxy = proxy_https_element
yield https_proxy
proxy_result = try_proxies(HTTP, HTTPS)
print(proxy_result)
# proxies = {
# 'http': http_proxy,
# 'https': https_proxy
# }
#
# try:
# res = requests.get("https://httpbin.org/ip", proxies=proxies, timeout=5)
# print(res)
# except requests.exceptions.ConnectTimeout:
# print("well at last your tried")
def main():
try_proxies(HTTP, HTTPS)
main()
but it doesn't work like I want it to work and I know that because it's not correct. How is the best way to do this?
Edit: I lowkey improved the code but still didn't get the result I expected because it doesn't test each proxy but the last value of https_proxy
import requests
HTTP = [List of HTTP proxies...]
HTTPS = [List of HTTPS proxies...]
def try_proxies(http_proxies, https_proxies):
for proxy_http_element in http_proxies:
http_proxy = proxy_http_element
for proxy_https_element in https_proxies:
https_proxy = proxy_https_element
proxies = {
'http': http_proxy,
'https': https_proxy,
}
try:
res = requests.get("https://httpbin.org/ip", proxies=proxies, timeout=20)
print("here")
print(res.status_code)
except requests.exceptions.ConnectTimeout:
print("well at last your tried")
def main():
try_proxies(HTTP, HTTPS)
main()

I solved the issue in this way:
import requests
HTTP = [List of http proxies...]
HTTPS = [List of https proxies]
def try_proxies(http_proxies, https_proxies):
for proxy_http_element in http_proxies:
http_proxy = proxy_http_element
yield http_proxy
for proxy_https_element in https_proxies:
https_proxy = proxy_https_element
yield https_proxy
test = [p for p in try_proxies(HTTP, HTTPS)]
for proxy in test:
print(f"{proxy}")
proxies = {
'http': proxy
}
try:
res = requests.get("https://httpbin.org/ip", proxies=proxies, timeout=10)
print("here")
print(res.status_code)
except requests.exceptions.ConnectTimeout as CT:
print(f"well at last your tried: {CT}")
I hope this can be helpful to someone else!
Edit: I removed the 'https': proxy because because of that I had an error when I removed that the proxies worked!

Setting proxy to urllib.request (Python3)

How can I set proxy for the last urllib in Python 3.
I am doing the next
from urllib import request as urlrequest
ask = urlrequest.Request(url) # note that here Request has R not r as prev versions
open = urlrequest.urlopen(req)
open.read()
I tried adding proxy as follows :
ask=urlrequest.Request.set_proxy(ask,proxies,'http')
However I don't know how correct it is since I am getting the next error:
336 def set_proxy(self, host, type):
--> 337 if self.type == 'https' and not self._tunnel_host:
338 self._tunnel_host = self.host
339 else:
AttributeError: 'NoneType' object has no attribute 'type'

You should be calling set_proxy() on an instance of class Request, not on the class itself:
from urllib import request as urlrequest
proxy_host = 'localhost:1234' # host and port of your proxy
url = 'http://www.httpbin.org/ip'
req = urlrequest.Request(url)
req.set_proxy(proxy_host, 'http')
response = urlrequest.urlopen(req)
print(response.read().decode('utf8'))

I needed to disable the proxy in our company environment, because I wanted to access a server on localhost. I could not disable the proxy server with the approach from #mhawke (tried to pass {}, None and [] as proxies).
This worked for me (can also be used for setting a specific proxy, see comment in code).
import urllib.request as request
# disable proxy by passing an empty
proxy_handler = request.ProxyHandler({})
# alertnatively you could set a proxy for http with
# proxy_handler = request.ProxyHandler({'http': 'http://www.example.com:3128/'})
opener = request.build_opener(proxy_handler)
url = 'http://www.example.org'
# open the website with the opener
req = opener.open(url)
data = req.read().decode('utf8')
print(data)

Urllib will automatically detect proxies set up in the environment - so one can just set the HTTP_PROXY variable either in your environment e.g. for Bash:
export HTTP_PROXY=http://proxy_url:proxy_port
or using Python e.g.
import os
os.environ['HTTP_PROXY'] = 'http://proxy_url:proxy_port'
Note from the urllib docs: "HTTP_PROXY[environment variable] will be ignored if a variable REQUEST_METHOD is set; see the documentation on getproxies()"

import urllib.request
def set_http_proxy(proxy):
if proxy == None: # Use system default setting
proxy_support = urllib.request.ProxyHandler()
elif proxy == '': # Don't use any proxy
proxy_support = urllib.request.ProxyHandler({})
else: # Use proxy
proxy_support = urllib.request.ProxyHandler({'http': '%s' % proxy, 'https': '%s' % proxy})
opener = urllib.request.build_opener(proxy_support)
urllib.request.install_opener(opener)
proxy = 'user:pass#ip:port'
set_http_proxy(proxy)
url = 'https://www.httpbin.org/ip'
request = urllib.request.Request(url)
response = urllib.request.urlopen(request)
html = response.read()
html

Logging into quora using python

I tried logging into quora using python. But it gives me the following error.
urllib2.HTTPError: HTTP Error 500: Internal Server Error
This is my code till now. I also work behind a proxy.
import urllib2
import urllib
import re
import cookielib
class Quora:
def __init__(self):
'''Initialising and authentication'''
auth = 'http://name:password#proxy:port'
cj = cookielib.CookieJar()
logindata = urllib.urlencode({'email' : 'email' , 'password' : 'password'})
handler = urllib2.ProxyHandler({'http' : auth})
opener = urllib2.build_opener(handler , urllib2.HTTPCookieProcessor(cj))
urllib2.install_opener(opener)
a = urllib2.urlopen('http://www.quora.com/' , logindata)
def main():
Quora()
Can someone please point out what is wrong?
if __name__ == '__main__':
main()

Try something like this:
# Load proxies
proxies = []
proxies_fp = open('proxies.txt', 'r') # A list of proxies
for proxy in proxies_fp:
proxies.append(proxy)
cookiejar = cookielib.CookieJar()
def perform_request(url, opener, credientials):
# Instantiate our request object
request = urllib2.Request(url)
# Perform the request, returning a pointer to the result set.
result = opener.urlopen(request, credentials)
return result
credentials ={
'username' : 'username',
'password' : 'password'
}
encoded_credentials = urllib.urlencode(credentials)
def main():
# Get random proxy
proxy = random.choice(proxies)
# Install our proxy
opener = urllib2.build_opener(
urllib2.ProxyHandler({'http': proxy}),
urllib2.HTTPRedirectHandler(),
urllib2.HTTPHandler(debuglevel=0),
urllib2.HTTPSHandler(debuglevel=0),
urllib2.HTTPCookieProcessor(cookiejar),
)
urllib2.install_opener(opener)
a = perform_request(url, opener, encoded_credentials)
--untested--
I've had to do something similar to this, and it worked for me this way. (Please note, that this is NOT an exact copy of code I used. I had to manipulate it a bit, and did NOT test)

How can I open a website with urllib via proxy in Python?

I have this program that check a website, and I want to know how can I check it via proxy in Python...
this is the code, just for example
while True:
try:
h = urllib.urlopen(website)
break
except:
print '['+time.strftime('%Y/%m/%d %H:%M:%S')+'] '+'ERROR. Trying again in a few seconds...'
time.sleep(5)

By default, urlopen uses the environment variable http_proxy to determine which HTTP proxy to use:
$ export http_proxy='http://myproxy.example.com:1234'
$ python myscript.py # Using http://myproxy.example.com:1234 as a proxy
If you instead want to specify a proxy inside your application, you can give a proxies argument to urlopen:
proxies = {'http': 'http://myproxy.example.com:1234'}
print("Using HTTP proxy %s" % proxies['http'])
urllib.urlopen("http://www.google.com", proxies=proxies)
Edit: If I understand your comments correctly, you want to try several proxies and print each proxy as you try it. How about something like this?
candidate_proxies = ['http://proxy1.example.com:1234',
'http://proxy2.example.com:1234',
'http://proxy3.example.com:1234']
for proxy in candidate_proxies:
print("Trying HTTP proxy %s" % proxy)
try:
result = urllib.urlopen("http://www.google.com", proxies={'http': proxy})
print("Got URL using proxy %s" % proxy)
break
except:
print("Trying next proxy in 5 seconds")
time.sleep(5)

Python 3 is slightly different here. It will try to auto detect proxy settings but if you need specific or manual proxy settings, think about this kind of code:
#!/usr/bin/env python3
import urllib.request
proxy_support = urllib.request.ProxyHandler({'http' : 'http://user:pass#server:port',
'https': 'https://...'})
opener = urllib.request.build_opener(proxy_support)
urllib.request.install_opener(opener)
with urllib.request.urlopen(url) as response:
# ... implement things such as 'html = response.read()'
Refer also to the relevant section in the Python 3 docs

Here example code guide how to use urllib to connect via proxy:
authinfo = urllib.request.HTTPBasicAuthHandler()
proxy_support = urllib.request.ProxyHandler({"http" : "http://ahad-haam:3128"})
# build a new opener that adds authentication and caching FTP handlers
opener = urllib.request.build_opener(proxy_support, authinfo,
urllib.request.CacheFTPHandler)
# install it
urllib.request.install_opener(opener)
f = urllib.request.urlopen('http://www.google.com/')
"""

For http and https use:
proxies = {'http':'http://proxy-source-ip:proxy-port',
'https':'https://proxy-source-ip:proxy-port'}
more proxies can be added similarly
proxies = {'http':'http://proxy1-source-ip:proxy-port',
'http':'http://proxy2-source-ip:proxy-port'
...
}
usage
filehandle = urllib.urlopen( external_url , proxies=proxies)
Don't use any proxies (in case of links within network)
filehandle = urllib.urlopen(external_url, proxies={})
Use proxies authentication via username and password
proxies = {'http':'http://username:password#proxy-source-ip:proxy-port',
'https':'https://username:password#proxy-source-ip:proxy-port'}
Note: avoid using special characters such as :,# in username and passwords

How to specify an authenticated proxy for a python http connection?

What's the best way to specify a proxy with username and password for an http connection in python?

This works for me:
import urllib2
proxy = urllib2.ProxyHandler({'http': 'http://
username:password#proxyurl:proxyport'})
auth = urllib2.HTTPBasicAuthHandler()
opener = urllib2.build_opener(proxy, auth, urllib2.HTTPHandler)
urllib2.install_opener(opener)
conn = urllib2.urlopen('http://python.org')
return_str = conn.read()

Use this:
import requests
proxies = {"http":"http://username:password#proxy_ip:proxy_port"}
r = requests.get("http://www.example.com/", proxies=proxies)
print(r.content)
I think it's much simpler than using urllib. I don't understand why people love using urllib so much.

Setting an environment var named http_proxy like this: http://username:password#proxy_url:port

The best way of going through a proxy that requires authentication is using urllib2 to build a custom url opener, then using that to make all the requests you want to go through the proxy. Note in particular, you probably don't want to embed the proxy password in the url or the python source code (unless it's just a quick hack).
import urllib2
def get_proxy_opener(proxyurl, proxyuser, proxypass, proxyscheme="http"):
password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, proxyurl, proxyuser, proxypass)
proxy_handler = urllib2.ProxyHandler({proxyscheme: proxyurl})
proxy_auth_handler = urllib2.ProxyBasicAuthHandler(password_mgr)
return urllib2.build_opener(proxy_handler, proxy_auth_handler)
if __name__ == "__main__":
import sys
if len(sys.argv) > 4:
url_opener = get_proxy_opener(*sys.argv[1:4])
for url in sys.argv[4:]:
print url_opener.open(url).headers
else:
print "Usage:", sys.argv[0], "proxy user pass fetchurls..."
In a more complex program, you can seperate these components out as appropriate (for instance, only using one password manager for the lifetime of the application). The python documentation has more examples on how to do complex things with urllib2 that you might also find useful.

Or if you want to install it, so that it is always used with urllib2.urlopen (so you don't need to keep a reference to the opener around):
import urllib2
url = 'www.proxyurl.com'
username = 'user'
password = 'pass'
password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
# None, with the "WithDefaultRealm" password manager means
# that the user/pass will be used for any realm (where
# there isn't a more specific match).
password_mgr.add_password(None, url, username, password)
auth_handler = urllib2.HTTPBasicAuthHandler(password_mgr)
opener = urllib2.build_opener(auth_handler)
urllib2.install_opener(opener)
print urllib2.urlopen("http://www.example.com/folder/page.html").read()

Here is the method use urllib
import urllib.request
# set up authentication info
authinfo = urllib.request.HTTPBasicAuthHandler()
proxy_support = urllib.request.ProxyHandler({"http" : "http://ahad-haam:3128"})
# build a new opener that adds authentication and caching FTP handlers
opener = urllib.request.build_opener(proxy_support, authinfo,
urllib.request.CacheFTPHandler)
# install it
urllib.request.install_opener(opener)
f = urllib.request.urlopen('http://www.python.org/')
"""