Like if I had a list of proxies how can I incorporate it to open the tab with it.
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
driver = webdriver.Chrome()
driver.get('https://www.youtube.com/watch?v=QggJzZdIYPI&ab_channel=CandRfun')
#using this link as an example.
The just replace the ip of your proxy and port with desired ip and desired port in 2nd line.
from selenium import webdriver
PROXY = "The desired IP:Desired port"
chrome_options = WebDriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=%s' % PROXY)
chrome = webdriver.Chrome(chrome_options=chrome_options)
chrome.get("https://www.youtube.com/watch?v=QggJzZdIYPI&ab_channel=CandRfun")
I'm trying to use a proxy within Google Cloud Engine with chromedriver.
I've tried many solutions suggested (see below) but everytime the IP was the one on Google server.
Attempt 1:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--window-size=1920x1080")
chrome_options.add_argument("--ignore-certificate-errors")
myproxy = '207.157.25.44:80'
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = myproxy
prox.ssl_proxy = myproxy
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
driver = webdriver.Chrome(options=chrome_options,
executable_path="/user/sebastien/chromedriver",
desired_capabilities=capabilities)
driver.get("https://www.whatismyip.com/")
get_location()
Attempt 2:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--window-size=1920x1080")
chrome_options.add_argument("--ignore-certificate-errors")
myproxy = '207.157.25.44:80'
prefs = {}
prefs["network.proxy.type"] = 1
prefs["network.proxy.http"] = myproxy
prefs["network.proxy.ssl"] = myproxy
chrome_options.add_experimental_option('prefs', prefs)
driver = webdriver.Chrome(options=chrome_options,
executable_path="/user/sebastien/chromedriver")
driver.get("https://www.whatismyip.com/")
get_location()
Attempt 3:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--window-size=1920x1080")
chrome_options.add_argument("--ignore-certificate-errors")
myproxy = '207.157.25.44:80'
chrome_options.add_argument("--proxy-server=http://%s" % myproxy)
driver = webdriver.Chrome(options=chrome_options,
executable_path="/user/sebastien/chromedriver")
driver.get("https://www.whatismyip.com/")
get_location()
None of them would reach the website with the desired IP.
Again, this issue is happening when running the code on GCP Compute Engine, Canonical, Ubuntu, 16.04 LTS, amd64 xenial.
Below the function to test the IP:
import json
from urllib.request import urlopen
def get_location(ip=False):
if ip:
html = urlopen(f"http://ipinfo.io/{str(ip).split(':')[0]}/json")
else:
html = urlopen("http://ipinfo.io/json")
data = json.loads(html.read().decode('utf-8'))
IP = data['ip']
org = data['org']
city = data['city']
country = data['country']
region = data['region']
print('IP detail')
print('IP : {4} \nRegion : {1} \nCountry : {2} \nCity : {3} \nOrg : {0}'.format(org, region, country, city, IP))
Thanks for reading !
I don't think the issue that you're having is related to your code implementation. I'm sure that the issue that you're having is related to your usage of a free proxy. These type of proxies
are notorious for having connections issues, such as timeouts related to latency. Plus these sites can also be intermittent, which means that they can go down at anytime. And sometimes these sites are being abused, so they can get blocked.
Your proxy is 207.157.25.44:80, which is shown in the image below.
When I tested this code:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
proxy_server = '207.157.25.44:80'
chrome_options = Options()
chrome_options.add_argument("--disable-infobars")
chrome_options.add_argument("start-maximized")
chrome_options.add_argument("--disable-extensions")
chrome_options.add_argument("--disable-popup-blocking")
chrome_options.add_argument('--proxy-server=%s' % proxy_server)
# disable the banner "Chrome is being controlled by automated test software"
chrome_options.add_experimental_option("useAutomationExtension", False)
chrome_options.add_experimental_option("excludeSwitches", ['enable-automation'])
driver = webdriver.Chrome('/usr/local/bin/chromedriver', options=chrome_options)
driver.get('https://www.whatismyip.com/')
The Chrome browser opens, but it does not display any content.
If I check the address 207.157.25.44:80 via an online proxy checker service, I get mixed results.
This image below shows that the proxy is not responding to any query types (HTTP, HTTPS, SOCKS4, SOCKS5).
When I do the same check 5 minutes later the proxy is up on HTTP, but has latency issues.
If I selected another proxy from the free proxy website:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
proxy_server = '47.184.133.79:3128'
chrome_options = Options()
chrome_options.add_argument("--disable-infobars")
chrome_options.add_argument("start-maximized")
chrome_options.add_argument("--disable-extensions")
chrome_options.add_argument("--disable-popup-blocking")
chrome_options.add_argument('--proxy-server=%s' % proxy_server)
# disable the banner "Chrome is being controlled by automated test software"
chrome_options.add_experimental_option("useAutomationExtension", False)
chrome_options.add_experimental_option("excludeSwitches", ['enable-automation'])
driver = webdriver.Chrome('/usr/local/bin/chromedriver', options=chrome_options)
driver.get('https://www.whatismyip.com/')
I get a CloudFlare challenge page when connecting to the website whatismyip.
But if I try the same proxy on the website nordvpn.com/what-is-my-ip I get the proxy's IP address.
I would highly recommend testing any free proxy IP address multiple times to see if the address has any types of issues. Additionally, you need to add some error handling in your code to catch issues when a proxy goes offline, because they can drop at anytime.
If you need to use a proxy, I would strongly recommend using a commercial proxy service, because they are more reliable than the free proxy services.
oxylabs.io
bright data
Is there a way to change the proxy ip address for your Selenium Chrome web driver without closing the browser and just by reloading?
I know that you can add a proxy by running this but how can you change it?:
from selenium import webdriver
PROXY = "23.23.23.23:3128" # IP:PORT or HOST:PORT
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=%s' % PROXY)
chrome = webdriver.Chrome(options=chrome_options)
I am trying to use Chrome Driver with proxy first time but it does not seem to work. Below is my code:
from selenium import webdriver
proxy = '1.1.1.1'
proxy_url = 'https://usr:pwd#' + proxy.strip().rstrip('\n')
print(proxy_url)
service = webdriver.chrome.service.Service(CHROME_PATH)
service.start()
options = webdriver.ChromeOptions()
options.add_argument('--headless')
options.add_argument('--proxy-server=%s' % proxy_url)
options = options.to_capabilities()
driver = webdriver.Remote(service.service_url, options)
driver.set_window_size(1400, 1000)
driver.get("http://google.com")
print(driver.page_source)
No matter what site I use it returns:
<html xmlns="http://www.w3.org/1999/xhtml"><head></head><body></body></html>
Sorry I can't share original proxy as they don't belong to me. Can anyone help me out?
My python script has the headless chrome in selenium up and functional but how, if possible, can I use a proxy as well? How can I pass the proxy host port to my headless chrome browser?
options = webdriver.ChromeOptions()
options.add_argument('headless')
browser = webdriver.Chrome(chrome_options=options)
How do I use the proxy host port with selenium and mainly headless chrome? Thanks!
Something like this:
from selenium import webdriver
from selenium.webdriver.common.proxy import Proxy, ProxyType
options = webdriver.ChromeOptions()
options.add_argument('headless')
desired_caps = options.to_capabilities()
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = "ip_addr:port"
prox.socks_proxy = "ip_addr:port"
prox.ssl_proxy = "ip_addr:port"
prox.add_to_capabilities(desired_caps)
browser = webdriver.Chrome(desired_capabilities=desired_caps)
The simplest way to do is something like below. Put any ip address in the proxy variable to find out how it works. ip and port are separated by :.
from selenium import webdriver
proxy = "94.122.251.105:3128" #test with any ip address which supports `http` as well because the link within the script are of `http`
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--proxy-server={}'.format(proxy))
driver = webdriver.Chrome(chrome_options=chrome_options)
driver.get('http://www.lagado.com/proxy-test')
items = driver.find_element_by_css_selector(".main-panel p:nth-of-type(2)").text
print(items) #it should print out the ip address you are using in the `proxy` variable above
driver.quit()