I'm using Firefox headless like this:
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
from selenium import webdriver
import os
import sys
# Set the MOZ_HEADLESS environment variable which casues Firefox to
# start in headless mode.
os.environ['MOZ_HEADLESS'] = '1'
# Select your Firefox binary.
binary = FirefoxBinary('/usr/bin/firefox', log_file=sys.stdout)
# Start selenium with the configured binary.
driver = webdriver.Firefox(firefox_binary=binary)
But now I want to add a http proxy that requires a user/password. After searching around, I tried the following:
from selenium.webdriver.common.proxy import Proxy, ProxyType
myProxy = "xx.xx.xx.xx:80"
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': myProxy,
'ftpProxy': myProxy,
'sslProxy': myProxy,
'noProxy': '' # set this value as desired
})
driver = webdriver.Firefox(firefox_binary=binary, proxy=proxy)
I also tried
profile = webdriver.FirefoxProfile()
profile.set_preference("network.proxy.type", 1)
profile.set_preference("network.proxy.http", "xx.xx.xx.xx")
profile.set_preference("network.proxy.http_port", 80)
profile.update_preferences()
driver=webdriver.Firefox(firefox_binary=binary,firefox_profile=profile)
Finally, I tried adding "socksUsername" and "socksPassword" with the creds to the proxy, more out of desperation than any real hope.
Needless to say none of these works, and testing shows requests are still using my usual IP, not the proxy.
Also system-wide proxy is not an option in this case.
Where should the http proxy credentials live? How can I use a proxy with headless firefox?
Testing
driver.get("https://www.ipinfo.io");
driver.find_element_by_xpath('//h4/following-sibling::p').text
If your proxy requires a username and a password, you have to write it like that :
from selenium.webdriver.common.proxy import Proxy, ProxyType
myProxy = "username:password#proxyDomain:proxyPort"
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': myProxy,
'ftpProxy': myProxy,
'sslProxy': myProxy,
'noProxy': '' # set this value as desired
})
driver = webdriver.Firefox(firefox_binary=binary, proxy=proxy)
Have you try setting up a profile manually with
./firefox --ProfileManager
manually setup the proxy and then load the profile you manually set
from selenium import webdriver
url = "https://mail.google.com"
fp = webdriver.FirefoxProfile('/Users/<username>/Library/Application Support/Firefox/Profiles/71v1uczn.default')
driver = webdriver.Firefox(fp)
You can try setting up an Environment variable "HTTP_PROXY" in following mnemonics:
http://<username>:<password>#<proxy_url>
Add your credentials separated by colon ':' before proxy url that is preceded by '#' for e.g.
http://username:password#proxy.com:8080/file.pac
Related
I have installed mitmproxy. It works fine when setting the proxy in my network preferences on my mac. I'd like to not have to set it up on my system but just for the traffic of a python script that uses Selenium, but it doesn't work:
from selenium import webdriver
from selenium.webdriver.common.proxy import *
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager
import time
def main():
options = Options()
#options.accept_untrusted_certs = True
#options.add_argument("--headless")
myProxy = "localhost:8080"
options.proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': myProxy,
'sslProxy': myProxy,
'noProxy': ''
})
driver = webdriver.Chrome(
ChromeDriverManager().install()
,options=options
)
driver.get('http://mitm.it')
time.sleep(20)
driver.quit()
main()
When the script opens http://mitm.it I get the following message:
"If you're seeing this it means your traffic is not passing through mitmproxy."
I guess the options are not passed correctly in the webdriver but I don't know why.
capabilities = webdriver.DesiredCapabilities.CHROME
Proxy.add_to_capabilities(capabilities)
driver = webdriver.chrome('tmp/chromedriver', desired_capabilities=capabilities)
driver.get("https://www.google.com/")`
I'm trying to open a chrome tab with proxy (no authentication needed) but I got this error:
add_to_capabilities() missing 1 required positional argument: 'capabilities' error
Try following code as EX:
from selenium import webdriver
from selenium.webdriver.common.proxy import *
proxy_url = "127.0.0.1:9009"
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': proxy_url,
'sslProxy': proxy_url,
'noProxy': ''})
capabilities = webdriver.DesiredCapabilities.CHROME
proxy.add_to_capabilities(capabilities)
driver = webdriver.Chrome(desired_capabilities=capabilities)
driver.get("http://example.org")
I want to use proxy with username and password in my docker selenium container.
I tried every solution that I found on so and it doesn't work. I tried http proxy:
capabilities = DesiredCapabilities.FIREFOX
capabilities['proxy'] = {
'proxyType': 'MANUAL',
'httpProxy': f'{proxy.ip_address}:{proxy.port}',
'sslProxy': 'ip:port',
'socksUsername': proxy.login,
'socksPassword': proxy.password
}
browser = webdriver.Remote(command_executor='http://hub:4444/wd/hub',
desired_capabilities=capabilities,
browser_profile=profile)
selenium.common.exceptions.InvalidArgumentException: Message: Invalid
proxy configuration entry: socksPassword
tried socks proxy:
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'socksProxy': f'{proxy.ip_address}:{proxy.port}',
'socksUsername': proxy.login,
'socksPassword': proxy.password
})
browser = webdriver.Remote(command_executor='http://hub:4444/wd/hub',
desired_capabilities=capabilities,
browser_profile=profile,
proxy=proxy)
Message: Invalid proxy configuration entry: socksPassword
I also tried to set proxy via firefox profile, like here, but it seems not working because there is no option to set password for proxy.
Proxy is working because when i do request:
proxy = 'socks5://username:password#ip:port' resp = requests.get('https://api.ipify.org?format=json',
proxies=dict(http=proxy,
https=proxy)) print(resp.json())
returns correct result
The only solution is using proxy without password
Python3. You need to pip3 install geckodriver
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.proxy import Proxy, ProxyType
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
firefox_capabilities = webdriver.DesiredCapabilities.FIREFOX
firefox_capabilities['marionette'] = True
# https://free-proxy-list.net/
proxy = '78.96.125.24:3128'
firefox_capabilities['proxy'] = {
"proxyType": "MANUAL",
"httpProxy": proxy,
"ftpProxy": proxy,
"sslProxy": proxy,
}
browser = webdriver.Firefox(capabilities=firefox_capabilities)
browser.get('https://httpbin.org/ip')
# browser.get('https://www.google.pl')
I am coding a test suite using Python and the Selenium library. Using the chromedriver, I am setting proxies using:
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=%s' % hostname + ":" + port)
global driver
driver = webdriver.Chrome(chrome_options=chrome_options)
This works fine when the proxy does not have authentication. However, if the proxy requires you to login with a username and password it will not work. What is the correct and proper way to pass proxy authentication information to the chromedriver using add_argument or other methods?
It is not the same as: How to set Proxy setting for Chrome in Selenium Java
Seeing as:
I ts a different language
Its firefox, not chrome.
--proxy-server=http://user:password#proxy.com:8080 does not work.
Use DesiredCapabilities. I have been successfully using proxy authentication with the following:
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
proxy = {'address': '123.123.123.123:2345',
'username': 'johnsmith123',
'password': 'iliketurtles'}
capabilities = dict(DesiredCapabilities.CHROME)
capabilities['proxy'] = {'proxyType': 'MANUAL',
'httpProxy': proxy['address'],
'ftpProxy': proxy['address'],
'sslProxy': proxy['address'],
'noProxy': '',
'class': "org.openqa.selenium.Proxy",
'autodetect': False}
capabilities['proxy']['socksUsername'] = proxy['username']
capabilities['proxy']['socksPassword'] = proxy['password']
driver = webdriver.Chrome(executable_path=[path to your chromedriver], desired_capabilities=capabilities)
EDIT: it unfortunately seems this method no longer works since one of the updated to either Selenium or Chrome since this post. as of now, i do not know another solution, but i will experiment and update this if i find anything out.
I have been using selenium for automatic browser simulations and web scraping in python and it has worked well for me. But now, I have to run it behind a proxy server. So now selenium open up the window but could not open the requested page because of proxy settings not set on the opened browser. Current code is as follows (sample):
from selenium import webdriver
sel = webdriver.Firefox()
sel.get('http://www.google.com')
sel.title
sel.quit()
How do I change the above code to work with proxy server now as well?
You need to set desired capabilities or browser profile, like this:
profile = webdriver.FirefoxProfile()
profile.set_preference("network.proxy.type", 1)
profile.set_preference("network.proxy.http", "proxy.server.address")
profile.set_preference("network.proxy.http_port", "port_number")
profile.update_preferences()
driver = webdriver.Firefox(firefox_profile=profile)
Also see related threads:
how do i set proxy for chrome in python webdriver
Selenium using Python: enter/provide http proxy password for firefox
Running Selenium Webdriver with a proxy in Python
http://krosinski.blogspot.ru/2012/11/selenium-firefox-webdriver-and-proxies.html
The official Selenium documentation (http://docs.seleniumhq.org/docs/04_webdriver_advanced.jsp#using-a-proxy) provides clear and helpful guidelines about using a proxy.
For Firefox (which is the browser of choice in your sample code) you should do the following:
from selenium import webdriver
from selenium.webdriver.common.proxy import *
myProxy = "host:8080"
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': myProxy,
'ftpProxy': myProxy,
'sslProxy': myProxy,
'noProxy': '' # set this value as desired
})
driver = webdriver.Firefox(proxy=proxy)
This will do the job:
import selenium
from selenium.webdriver.common.proxy import *
proxyHost = "my.proxy.host or IP"
proxyPort = "55555"
fp = webdriver.FirefoxProfile()
fp.set_preference("network.proxy.type", 1)
#fp.set_preference("network.proxy.http", proxyHost) #HTTP PROXY
#fp.set_preference("network.proxy.http_port", int(proxyPort))
#fp.set_preference("network.proxy.ssl", proxyHost) #SSL PROXY
#fp.set_preference("network.proxy.ssl_port", int(proxyPort))
fp.set_preference('network.proxy.socks', proxyHost) #SOCKS PROXY
fp.set_preference('network.proxy.socks_port', int(proxyPort))
fp.update_preferences()
driver = webdriver.Firefox(firefox_profile=fp)
driver.get("http://www.whatismyip.com/")
def install_proxy(PROXY_HOST,PROXY_PORT):
fp = webdriver.FirefoxProfile()
print PROXY_PORT
print PROXY_HOST
fp.set_preference("network.proxy.type", 1)
fp.set_preference("network.proxy.http",PROXY_HOST)
fp.set_preference("network.proxy.http_port",int(PROXY_PORT))
fp.set_preference("network.proxy.https",PROXY_HOST)
fp.set_preference("network.proxy.https_port",int(PROXY_PORT))
fp.set_preference("network.proxy.ssl",PROXY_HOST)
fp.set_preference("network.proxy.ssl_port",int(PROXY_PORT))
fp.set_preference("network.proxy.ftp",PROXY_HOST)
fp.set_preference("network.proxy.ftp_port",int(PROXY_PORT))
fp.set_preference("network.proxy.socks",PROXY_HOST)
fp.set_preference("network.proxy.socks_port",int(PROXY_PORT))
fp.set_preference("general.useragent.override","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A")
fp.update_preferences()
return webdriver.Firefox(firefox_profile=fp)