I'm currently developping a web app using django and the App platform on digitalocean.
My web app uses selenium and chromedriver, I have found a way to install chromedriver using python libs such as chromedriver_binary on pip but the app can't open it and throws me an error :
Message: Service chromedriver unexpectedly exited. Status code was: 127
This error most likely means that some dependencies and libs are not available and leads to the script crashing.
Here's my current code inside views.py :
from selenium import webdriver
import chromedriver_binary
import time
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium_stealth import stealth
import undetected_chromedriver as uc
from selenium.webdriver.chromium.options import ChromiumOptions
from selenium.webdriver.common.by import By
import urllib.request
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import UnexpectedAlertPresentException
from selenium.common.exceptions import NoSuchElementException
from googleapiclient.discovery import build
from urllib.parse import urlsplit
def get_webdriver(url):
options = webdriver.ChromeOptions()
options.add_argument('--disable-extensions')
options.add_argument('--headless')
options.add_argument('--disable-gpu')
options.add_argument('--no-sandbox')
options.add_argument('start-maximized')
options.add_argument('disable-infobars')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--disable-browser-side-navigation')
options.add_argument('--disable-gpu')
driver = webdriver.Chrome()
driver.get(url)
return driver
Is there a way (maybe using another lib, or a whole other solution) to deploy chromedriver on this django app?
EDIT 1 :
Seems like the app does find the chromedriver, I can print it through the code below :
def get_webdriver(url):
from selenium.webdriver.chrome.service import Service
driver_path = chromedriver_binary.chromedriver_filename
print("Driver's path : ", driver_path)
ser = Service(driver_path)
options = webdriver.ChromeOptions()
options.add_argument('--headless')
driver = webdriver.Chrome(service=ser)
driver.get(url)
return driver
The print returns me :
/workspace/.heroku/python/lib/python3.10/site-packages/chromedriver_binary/chromedriver
It really seems like some libraries or dependencies are missing but I don't know how to install them on this app platform.
Remove the unwanted arguments unless any specific requirement:
options.add_argument('--disable-extensions')
options.add_argument('--disable-gpu')
options.add_argument('--no-sandbox')
options.add_argument('start-maximized')
options.add_argument('disable-infobars')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--disable-browser-side-navigation')
options.add_argument('--disable-gpu')
and keep only the intended one:
options.add_argument('--headless')
and execute your test.
Related
I test selenium python and I have these errors with firefox and I can't find the solution. Here my code
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
options = Options()
options.binary = FirefoxBinary(r'/usr/bin/firefox')
driver = webdriver.Firefox(executable_path= r'./scrap/geckodriver', options=options)
driver.get("http://www.google.com")
print (driver.page_source.encode('utf-8') )
driver.close()
My error message python linuxmint :
selenium.common.exceptions.InvalidArgumentException: Message: binary is not a Firefox executable
I've been searching the forum for a few days, but I can't find a solution for linux mint.
This is an example of working selenium/Firefox setup on Debian, which should also work on Mint, given it's essentially the same os:
from selenium import webdriver
from selenium.webdriver.firefox.service import Service
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.firefox.options import Options as Firefox_Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support import expected_conditions as EC
import time as t
import pandas as pd
firefox_options = Firefox_Options()
# firefox_options.add_argument("--width=1280")
# firefox_options.add_argument("--height=720")
driverService = Service('chromedriver/geckodriver') ## path where you saved geckodriver
browser = webdriver.Firefox(service=driverService, options=firefox_options)
Selenium docs: https://www.selenium.dev/documentation/
As an alternative to options.binary = FirefoxBinary() you can use the binary_location option as follows:
options = Options()
options.binary_location = r'/usr/bin/firefox'
I'm trying to get a html from the url below.
https://reserve.tokyodisneyresort.jp/restaurant/search/?useDate=20220714&adultNum=4&childNum=0&childAgeInform=&restaurantType=4&restaurantType=5&restaurantType=3&restaurantType=1&restaurantType=2&restaurantType=7&nameCd=&wheelchairCount=0&stretcherCount=0&keyword=&reservationStatus=0
But after starting chromedriver, it continues to load all the time and nothing appears.
How can I fix it?
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
import time
def get_html():
url = 'https://reserve.tokyodisneyresort.jp/restaurant/search/?useDate=20220714&adultNum=4&childNum=0&childAgeInform=&restaurantType=4&restaurantType=5&restaurantType=3&restaurantType=1&restaurantType=2&restaurantType=7&nameCd=&wheelchairCount=0&stretcherCount=0&keyword=&reservationStatus=0'
options = Options()
options.add_experimental_option('excludeSwitches', ['enable-logging'])
options.add_argument("--disable-blink-features")
options.add_argument("--disable-blink-features=AutomationControlled")
driver = webdriver.Chrome(
executable_path='./chromedriver.exe', chrome_options=options)
driver.implicitly_wait(60)
driver.get(url)
get_html()
I try to open the following site using selenium:
https://www.honestdoor.com/
Normally this works fine with every site with the following code:
(I am currently using google-chrome version 98.0.4758 - using ChromeDriverManager for downloading the version - see below in the code)
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from webdriver_manager.chrome import ChromeDriverManager
from fake_useragent import UserAgent
import time
if __name__ == '__main__':
ua = UserAgent()
userAgent = ua.random
options = Options()
# options.add_argument('--headless')
options.add_experimental_option ('excludeSwitches', ['enable-logging'])
options.add_argument("start-maximized")
options.add_argument('window-size=1920x1080')
options.add_argument('--no-sandbox')
options.add_argument('--disable-gpu')
options.add_argument(f'user-agent={userAgent}')
srv=Service(ChromeDriverManager().install())
driver = webdriver.Chrome (service=srv, options=options)
waitWebDriver = WebDriverWait (driver, 10)
link = "https://www.honestdoor.com/"
# link = "https://www.bcassessment.ca/"
# driver.minimize_window() # optional
driver.get (link)
time.sleep(1000)
The site opens with selenium as allways but then the site goes immediately complete white and is still loading forever with the cicle going around in the top left corner (I can only kill the chrome-task in the task manager).
When I open the site in normal chrome or incognito chrome everything works fine - it seem to only crash when I open it with selenium. With other sites (like https://www.bcassessment.ca/ I have no problems at all and the open with selenium as allways)
Why is this not working for this particular website?
Not that super clear about the exact issue you are facing while loading the website. However I was able to load the website using the following code block:
Code Block:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
options = Options()
options.add_argument("start-maximized")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('excludeSwitches', ['enable-logging'])
options.add_experimental_option('useAutomationExtension', False)
options.add_argument('--disable-blink-features=AutomationControlled')
s = Service('C:\\BrowserDrivers\\chromedriver.exe')
driver = webdriver.Chrome(service=s, options=options)
driver.get("https://www.honestdoor.com/")
Browser Snapshot:
I'm trying to run Selenium in AWS Lambda (Python) and recently learned that since Chromedriver version 2.31 they changed chromeOptions to goog:chromeOptions
https://chromedriver.storage.googleapis.com/2.31/notes.txt
https://www.selenium.dev/selenium/docs/api/rb/Selenium/WebDriver/Chrome/Options.html
I am running Chromium 86 with Selenium 3.14. I've been troubleshooting multiple error messages for the past hours and I suspect this is my underlying issue I just dont know how can I pass this argument to the driver or selenium. I've been trying the following with no luck
driver = webdriver.Chrome(options='goog:chromeOptions')
or
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('goog:chromeOptions')
EDIT
Here's a snippet of the code I'm using
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('headless')
chrome_options.add_argument('--disable-gpu')
driver = webdriver.Chrome(chrome_options=chrome_options)
driver.get('https://google.com')
Any help is greatly appreciated
You saw it right.
You have to add the following import:
from selenium.webdriver.chrome.options import Options
Now you can add any of the arguments. As an example:
options = Options()
options.add_argument("start-maximized")
And finally:
driver = webdriver.Chrome(options=options)
driver.get("https://www.google.com/")
As an alternative,
from selenium import webdriver
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("start-maximized")
driver = webdriver.Chrome(options=chrome_options)
driver.get("https://www.google.com/")
Update
ChromeOptions object: Most Chrome-specific capabilities are exposed through the ChromeOptions object. In some languages, this is implemented by the ChromeOptions class. In other languages, they are stored under the goog:chromeOptions dictionary in desired capabilities.
As an example, in Ruby:
caps = Selenium::WebDriver::Remote::Capabilities.chrome(
"goog:chromeOptions" => {"args" => [ "window-size=1000,800" ]})
driver = Selenium::WebDriver.for :chrome, desired_capabilities: caps
Hello I'm trying to install a Chrome extension with Selenium using python, I tried using ChromeDriver - WebDriver for Chrome
But it is not working, this is my code:
from selenium import webdriver
from selenium.webdriver import Chrome
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.options import ChromeOptions
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.expected_conditions import presence_of_element_located
import re # regular expressions, are imported from python directly
import time
import numpy as np
import pandas as pd
import functions_database
# Pandas read CSV
df_read = pd.read_csv(
'/home/daniel/amazon-project-scrapers/ss_scraper.edited2.csv')
amazon_data = list(df_read.amz_search)
# Chrome Driver + install plugin
ChromeOptions options = new ChromeOptions();
options.addExtensions(new File("/home/daniel/amazon-project-scrapers/chromedriver_linux64/DS-Amazon-Quick-View_v2.8.9.crx"));
ChromeDriver driver = new ChromeDriver(options);
driver = webdriver.Chrome(executable_path='/home/daniel/amazon-project-scrapers/chromedriver_linux64/chromedriver')
driver.get('https://www.amazon.com/')
And this is the error i'm getting:
File "camel_scraper.py", line 23
ChromeOptions options = new ChromeOptions();
^
SyntaxError: invalid syntax
I tried to do this in other 3 different ways, actually there is a similar question in Stack overflow whose answer is deprecated, if I find it again I'll write the link in here.
To add/install the DS-Amazon-Quick-View Chrome extension using Selenium's python client you can use the following splution:
Code Block:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_extension('/home/daniel/amazon-project-scrapers/chromedriver_linux64/DS-Amazon-Quick-View_v2.8.9.crx')
driver = webdriver.Chrome(options=chrome_options, executable_path='/path/to/chromedriver')
driver.get('https://www.google.co.in')
Reference
You can find a couple of relevant discussions in:
[Python] How to load extension within chrome driver in selenium with python
[Java] How to install extension permanently in geckodriver