from selenium import webdriver
firefox_capabilities = webdriver.DesiredCapabilities.FIREFOX
firefox_capabilities['marionette'] = True
proxy = "115.77.166.65:41710"
firefox_capabilities['proxy'] = {
"proxyType": "MANUAL",
"httpProxy": proxy,
"ftpProxy": proxy,
"sslProxy": proxy
}
driver = webdriver.Firefox(capabilities=firefox_capabilities)
driver.get("https://whoer.net/")
driver = webdriver.Firefox(capabilities=firefox_capabilities)
DeprecationWarning: capabilities and desired_capabilities have been deprecated, please pass in a Service object
How to fix?
I'm trying to run a python program that involves threads and a proxy list but not all these proxies might work. If a proxy failure happens how do I move on to the next proxy without using a proxy another thread has used?
My code:
from threading import Thread
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
def run_session(accounts,proxy):
firefox_capabilities = webdriver.DesiredCapabilities.FIREFOX
firefox_capabilities['proxy'] = {
"proxyType": "MANUAL",
"httpProxy": proxy,
"sslProxy": proxy
}
driver = webdriver.Firefox(capabilities=firefox_capabilities)
driver.get('example')
accounts = [
{
'test_name':'Test_1',
'name':'NA',
},
{
'test_name':'Test_2',
'name':'NA',
},
{
'test_name':'Test_3',
'name':'NA',
}]
perfect_proxies_list = [62.205.169.74:53281,66.94.97.238:443,178.217.172.206:55443,171.97.54.242:55443,119.76.142.153:8080]
for act, proxy in zip(accounts, perfect_proxies_list):
Thread(target=run_session, args=(act,proxy)).start()
I really can’t to set socks5 proxy(http too...) for my chrome webdriver in selenium for python.
I tried many different ways... But I think I do something bad.
Example 1:
self.options.add_argument('--proxy-server=http://'+proxy)
Example 2:
webdriver.DesiredCapabilities.CHROME['proxy'] = {
"socksProxy": proxy,
"ftpProxy": proxy,
"sslProxy": proxy,
"noProxy": None,
"proxyType": "MANUAL",
"class": "org.openqa.selenium.Proxy",
"autodetect": False
}
Please describe fully the working example of setting up socks5 proxy on Selenium for Python and Chrome webdriver, with an example of proxy string formats (maybe i am doing something mistakes here ...).
PS Two problems which I get:
Just staying old IP address.
No internet connection in chrome web driver.
Chrome do not allow proxy with auth. I am not shure but after read so many informations I think so.... Only one way is working for me - to use proxy socks5 without auth by login and password.
options = webdriver.ChromeOptions()
proxy = '12.12.421.125:1949'
options.add_argument('--proxy-server=socks5://' + proxy)
driver = webdriver.Chrome(options=options)
For FireFox's geckodriver if you just want to set socks5 host / socks5 proxy :-
form selenium import webdriver
profile = webdriver.FirefoxProfile()
# Socks5 Host SetUp:-
myProxy = "198.199.101.152:8388"
ip, port = myProxy.split(':')
profile.set_preference('network.proxy.type', 1)
profile.set_preference('network.proxy.socks', ip)
profile.set_preference('network.proxy.socks_port', int(port))
driver = webdriver.Firefox(firefox_profile=profile)
Here is the code I used to connect to a Socks5 server with username/password auth.
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
capabilities = dict(DesiredCapabilities.CHROME)
capabilities['proxy'] = {
'proxyType': 'MANUAL',
'socksProxy': '<Your_IP>:<Your_Port>',
'socksVersion': 5,
'ftpProxy': '<Your_IP>:<Your_Port>',
'noProxy': 'localhost,127.0.0.1',
'class': "org.openqa.selenium.Proxy",
'autodetect': False
}
capabilities['proxy']['socksUsername'] = '<username>'
capabilities['proxy']['socksPassword'] = '<password>'
driver = Chrome(ChromeDriverManager().install(), desired_capabilities=capabilities)
For Firefox's geckodriver , based on the answer by Tanmay Harsh, I have got this so far:
from selenium.webdriver import Firefox, FirefoxOptions
proxy = ('proxy-server.local', 1080)
options = FirefoxOptions()
options.set_preference('network.proxy.type', 1)
options.set_preference('network.proxy.socks', proxy[0])
options.set_preference('network.proxy.socks_port', proxy[1])
options.set_preference('network.proxy.socks_remote_dns', True)
driver = Firefox(options=options)
'proxy': {
'http': 'socks5://user:pass#192.168.10.100:8888',
'https': 'socks5://user:pass#192.168.10.100:8888',
'no_proxy': 'localhost,127.0.0.1'
}
}
driver = webdriver.Chrome(seleniumwire_options=options)```
source - https://github.com/wkeeling/selenium-wire#socks
I want to use proxy with username and password in my docker selenium container.
I tried every solution that I found on so and it doesn't work. I tried http proxy:
capabilities = DesiredCapabilities.FIREFOX
capabilities['proxy'] = {
'proxyType': 'MANUAL',
'httpProxy': f'{proxy.ip_address}:{proxy.port}',
'sslProxy': 'ip:port',
'socksUsername': proxy.login,
'socksPassword': proxy.password
}
browser = webdriver.Remote(command_executor='http://hub:4444/wd/hub',
desired_capabilities=capabilities,
browser_profile=profile)
selenium.common.exceptions.InvalidArgumentException: Message: Invalid
proxy configuration entry: socksPassword
tried socks proxy:
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'socksProxy': f'{proxy.ip_address}:{proxy.port}',
'socksUsername': proxy.login,
'socksPassword': proxy.password
})
browser = webdriver.Remote(command_executor='http://hub:4444/wd/hub',
desired_capabilities=capabilities,
browser_profile=profile,
proxy=proxy)
Message: Invalid proxy configuration entry: socksPassword
I also tried to set proxy via firefox profile, like here, but it seems not working because there is no option to set password for proxy.
Proxy is working because when i do request:
proxy = 'socks5://username:password#ip:port' resp = requests.get('https://api.ipify.org?format=json',
proxies=dict(http=proxy,
https=proxy)) print(resp.json())
returns correct result
The only solution is using proxy without password
Python3. You need to pip3 install geckodriver
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.proxy import Proxy, ProxyType
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
firefox_capabilities = webdriver.DesiredCapabilities.FIREFOX
firefox_capabilities['marionette'] = True
# https://free-proxy-list.net/
proxy = '78.96.125.24:3128'
firefox_capabilities['proxy'] = {
"proxyType": "MANUAL",
"httpProxy": proxy,
"ftpProxy": proxy,
"sslProxy": proxy,
}
browser = webdriver.Firefox(capabilities=firefox_capabilities)
browser.get('https://httpbin.org/ip')
# browser.get('https://www.google.pl')
I'm using this code:
profile = webdriver.FirefoxProfile()
profile.set_preference("network.proxy.type", 1)
profile.set_preference("network.proxy.http", "proxy.server.address")
profile.set_preference("network.proxy.http_port", "port_number")
profile.update_preferences()
driver = webdriver.Firefox(firefox_profile=profile)
to set proxy for FF in python webdriver. This works for FF. How to set proxy like this in Chrome? I found this exmaple but is not very helpful. When I run the script nothing happens (Chrome browser is not started).
from selenium import webdriver
PROXY = "23.23.23.23:3128" # IP:PORT or HOST:PORT
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=%s' % PROXY)
chrome = webdriver.Chrome(options=chrome_options)
chrome.get("http://whatismyipaddress.com")
Its working for me...
from selenium import webdriver
PROXY = "23.23.23.23:3128" # IP:PORT or HOST:PORT
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=http://%s' % PROXY)
chrome = webdriver.Chrome(chrome_options=chrome_options)
chrome.get("http://whatismyipaddress.com")
I had an issue with the same thing. ChromeOptions is very weird because it's not integrated with the desiredcapabilities like you would think. I forget the exact details, but basically ChromeOptions will reset to default certain values based on whether you did or did not pass a desired capabilities dict.
I did the following monkey-patch that allows me to specify my own dict without worrying about the complications of ChromeOptions
change the following code in /selenium/webdriver/chrome/webdriver.py:
def __init__(self, executable_path="chromedriver", port=0,
chrome_options=None, service_args=None,
desired_capabilities=None, service_log_path=None, skip_capabilities_update=False):
"""
Creates a new instance of the chrome driver.
Starts the service and then creates new instance of chrome driver.
:Args:
- executable_path - path to the executable. If the default is used it assumes the executable is in the $PATH
- port - port you would like the service to run, if left as 0, a free port will be found.
- desired_capabilities: Dictionary object with non-browser specific
capabilities only, such as "proxy" or "loggingPref".
- chrome_options: this takes an instance of ChromeOptions
"""
if chrome_options is None:
options = Options()
else:
options = chrome_options
if skip_capabilities_update:
pass
elif desired_capabilities is not None:
desired_capabilities.update(options.to_capabilities())
else:
desired_capabilities = options.to_capabilities()
self.service = Service(executable_path, port=port,
service_args=service_args, log_path=service_log_path)
self.service.start()
try:
RemoteWebDriver.__init__(self,
command_executor=self.service.service_url,
desired_capabilities=desired_capabilities)
except:
self.quit()
raise
self._is_remote = False
all that changed was the "skip_capabilities_update" kwarg. Now I just do this to set my own dict:
capabilities = dict( DesiredCapabilities.CHROME )
if not "chromeOptions" in capabilities:
capabilities['chromeOptions'] = {
'args' : [],
'binary' : "",
'extensions' : [],
'prefs' : {}
}
capabilities['proxy'] = {
'httpProxy' : "%s:%i" %(proxy_address, proxy_port),
'ftpProxy' : "%s:%i" %(proxy_address, proxy_port),
'sslProxy' : "%s:%i" %(proxy_address, proxy_port),
'noProxy' : None,
'proxyType' : "MANUAL",
'class' : "org.openqa.selenium.Proxy",
'autodetect' : False
}
driver = webdriver.Chrome( executable_path="path_to_chrome", desired_capabilities=capabilities, skip_capabilities_update=True )
It's easy!
First, define your proxy url
proxy_url = "127.0.0.1:9009"
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': proxy_url,
'sslProxy': proxy_url,
'noProxy': ''})
Then, create chrome capability settings and add proxy to them
capabilities = webdriver.DesiredCapabilities.CHROME
proxy.add_to_capabilities(capabilities)
Finally, create the webdriver and pass the desired capabilities
driver = webdriver.Chrome(desired_capabilities=capabilities)
driver.get("http://example.org")
All together it looks like this
from selenium import webdriver
from selenium.webdriver.common.proxy import *
proxy_url = "127.0.0.1:9009"
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': proxy_url,
'sslProxy': proxy_url,
'noProxy': ''})
capabilities = webdriver.DesiredCapabilities.CHROME
proxy.add_to_capabilities(capabilities)
driver = webdriver.Chrome(desired_capabilities=capabilities)
driver.get("http://example.org")
This worked for me like a charm:
proxy = "localhost:8080"
desired_capabilities = webdriver.DesiredCapabilities.CHROME.copy()
desired_capabilities['proxy'] = {
"httpProxy": proxy,
"ftpProxy": proxy,
"sslProxy": proxy,
"noProxy": None,
"proxyType": "MANUAL",
"class": "org.openqa.selenium.Proxy",
"autodetect": False
}
For people out there asking how to setup proxy server in chrome which needs authentication should follow these steps.
Create a proxy.py file in your project, use this code and call proxy_chrome from
proxy.py everytime you need it. You need to pass parameters like proxy server, port and username password for authentication.
from selenium import webdriver
from selenium.webdriver.common.proxy import Proxy
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.chrome.options import Options
import zipfile,os
def proxy_chrome(PROXY_HOST,PROXY_PORT,PROXY_USER,PROXY_PASS):
manifest_json = """
{
"version": "1.0.0",
"manifest_version": 2,
"name": "Chrome Proxy",
"permissions": [
"proxy",
"tabs",
"unlimitedStorage",
"storage",
"<all_urls>",
"webRequest",
"webRequestBlocking"
],
"background": {
"scripts": ["background.js"]
},
"minimum_chrome_version":"22.0.0"
}
"""
background_js = """
var config = {
mode: "fixed_servers",
rules: {
singleProxy: {
scheme: "https",
host: "%(host)s",
port: parseInt(%(port)d)
},
bypassList: ["foobar.com"]
}
};
chrome.proxy.settings.set({value: config, scope: "regular"}, function() {});
function callbackFn(details) {
return {
authCredentials: {
username: "%(user)s",
password: "%(pass)s"
}
};
}
chrome.webRequest.onAuthRequired.addListener(
callbackFn,
{urls: ["<all_urls>"]},
['blocking']
);
""" % {
"host": PROXY_HOST,
"port": PROXY_PORT,
"user": PROXY_USER,
"pass": PROXY_PASS,
}
pluginfile = 'extension/proxy_auth_plugin.zip'
with zipfile.ZipFile(pluginfile, 'w') as zp:
zp.writestr("manifest.json", manifest_json)
zp.writestr("background.js", background_js)
co = Options()
#extension support is not possible in incognito mode for now
#co.add_argument('--incognito')
co.add_argument('--disable-gpu')
#disable infobars
co.add_argument('--disable-infobars')
co.add_experimental_option("excludeSwitches",["ignore-certificate-errors"])
#location of chromedriver, please change it according to your project.
chromedriver = os.getcwd()+'/Chromedriver/chromedriver'
co.add_extension(pluginfile)
driver = webdriver.Chrome(chromedriver,chrome_options=co)
#return the driver with added proxy configuration.
return driver
from selenium import webdriver
from selenium.webdriver.common.proxy import *
myProxy = "86.111.144.194:3128"
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': myProxy,
'ftpProxy': myProxy,
'sslProxy': myProxy,
'noProxy':''})
driver = webdriver.Firefox(proxy=proxy)
driver.set_page_load_timeout(30)
driver.get('http://whatismyip.com')