I'm trying to automate a search on an e-commerce using selenium with chrome driver, from the first url, type the search, click on the search button, return the next url and that's it.
But after clicking, the next page won't load and I can't figure out why.
the code:
from selenium import webdriver
def open():
options = webdriver.ChromeOptions()
options.add_argument("--window-size=1920x1080")
options.add_argument("--no-sandbox")
options.add_argument("--disable-gpu")
drive = webdriver.Chrome(options=options)
url = 'https://www.extra.com.br/'
drive.get(url)
busca = drive.find_element_by_id('ctl00_TopBar_PaginaSistemaArea1_ctl05_ctl00_txtBusca')
busca.send_keys('Informática')
botao_buscar = drive.find_element_by_id('ctl00_TopBar_PaginaSistemaArea1_ctl05_ctl00_btnOK')
botao_buscar.click()
return drive.current_url
if __name__ == '__main__':
print(open())
I've tried just opening the second url directly (without the form submit) and it works.
Related
I am building a dynamic scraper with selenium and flask that can take in any URL and scrape for cookies and other details. Now I want to check if the URL has any cookie consent popup. I am unable to make this feature dynamic.
I have tried PARTIAL_LINK_TEXT, it works only for some website
url="https://www.spitzer-silo.com/"
desired_capabilities = DesiredCapabilities.CHROME
desired_capabilities["goog:loggingPrefs"] = {"performance": "ALL"}
options = webdriver.ChromeOptions()
options.add_argument('headless')
options.add_argument("--ignore-certificate-errors")
driver = webdriver.Chrome(ChromeDriverManager().install(),options=options, desired_capabilities=desired_capabilities)
driver.get(url)
myDiv = driver.find_element(By.PARTIAL_LINK_TEXT, 'Cookie')
https://www.spitzer-silo.com/ works
https://www.siemens.com/de/de.html doesn't work
Also, I am searching with the "Cookie" keyword, which may not be present on some websites
another approach, I tried using a window handle but it shows only one window
url="https://www.siemens.com/de/de.html"
desired_capabilities = DesiredCapabilities.CHROME
desired_capabilities["goog:loggingPrefs"] = {"performance": "ALL"}
# Create the webdriver object and pass the arguments
options = webdriver.ChromeOptions()
# Chrome will start in Headless mode
options.add_argument('headless')
# Ignores any certificate errors if there is any
options.add_argument("--ignore-certificate-errors")
# Startup the chrome webdriver with executable path and
# pass the chrome options and desired capabilities as
# parameters.
driver = webdriver.Chrome(ChromeDriverManager().install(),options=options, desired_capabilities=desired_capabilities)
# Send a request to the website and let it load
driver.get(url)
time.sleep(30)
whandle = driver.window_handles
['CDwindow-E9E6A9B1021BBA75132EF9DCA40A2824']
Is there any way I could check if there is a popup on the website and then check if the popup has a text cookie on it
I appreciate all the help I can get.
I am using Selenium webdriver to fill out a form, the fields are good, but when I do a .submit using the login button, or do a .submit after the password field, the chrome browser will say "Unable to connect to internet. Please check your internet connection."
the internet is definitely up since I am able to open other websites on another browser (including a new session of chrome).
This is my code:
import time
import selenium
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
chrome_options = webdriver.ChromeOptions();
chrome_options.add_experimental_option("excludeSwitches", ['enable-automation']);
options = Options()
# options.add_argument('start-maximumized')
# options.add_argument('disable-infobars')
chrome_options.add_argument('no-sandbox')
PATH = "c:\scripts\chromedriver.exe"
# driver = webdriver.Chrome(executable_path=PATH)
driver = webdriver.Chrome(chrome_options=chrome_options,
executable_path='c:\scripts\chromedriver.exe')
driver.get('https://stupidwebsite.org')
time.sleep(1)
input_username = driver.find_element_by_id('username')
time.sleep(1)
input_username.send_keys("myself#email.com")
input_password = driver.find_element_by_id('password')
input_password.send_keys("mypassword#123")
input_password.submit()
# esubmit=driver.find_element_by_xpath
('/html/body/div/div[1]/div[2]/div/div/form/div[2]/div/button')
# esubmit = driver.find_element_by_name('MuiButton-label')
# esubmit.submit()
time.sleep(2)
# driver.quit()
I changed it a bit, and tried out google, and it works:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
#set chromodriver.exe path
driver = webdriver.Chrome(executable_path="C:\scripts\chromedriver.exe")
driver.implicitly_wait(0.5)
#launch URL
driver.get("https://www.google.com/")
#identify search box
m = driver.find_element_by_name("q")
#enter search text
m.send_keys("Tutorialspoint")
time.sleep(0.2)
#perform Google search with Keys.ENTER
m.send_keys(Keys.ENTER)
Please help me out. Thank you.
I have tried the following code and tried to open the website as mentioned:
driver = webdriver.Chrome(r"..\chromedriver_win32\chromedriver.exe")
driver.get("https://example.com")
The website opens with the Chrome Browser but not with the Selenium using Python.
Please let me know what should I do to open the website completely.
You can run it with chrome options. I am able to launch your application with below code:
from time import sleep
from selenium import webdriver
PATH = "chromedriver path"
option = webdriver.ChromeOptions()
option.add_argument('--disable-blink-features=AutomationControlled')
option.add_argument("start-maximized")
option.add_experimental_option(
"excludeSwitches", ["enable-automation"])
option.add_experimental_option('useAutomationExtension', False)
driver = webdriver.Chrome(PATH, options=option)
url = 'https://example.com'
driver.get(url)
driver.maximize_window()
sleep(20)
output:
I am using selenium to open and sign into google accounts as my first step. I have successfully opened and filled the email response although upon submitting I receive the error of
"This browser or app may not be secure. Learn more Try using a
different browser. If you’re already using a supported browser, you
can refresh your screen and try again to sign in." From google.
Is there any way to get around this? Here is my code below.
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
PATH = "C:\Program Files (x86)\chromedriver.exe"
driver = webdriver.Chrome(PATH)
driver.get("https://accounts.google.com/")
print(driver.title)
search = driver.find_element_by_name("identifier")
search.send_keys("email goes here")
search.send_keys(Keys.RETURN)
Was having the same issue and i found this thread in GitHub.
The solution that worked for me was to use this driver: undetected_chromedriver instead of the normal ChromeDriver.
import undetected_chromedriver.v2 as uc
chrome_options = uc.ChromeOptions()
chrome_options.add_argument("--disable-extensions")
chrome_options.add_argument("--disable-popup-blocking")
chrome_options.add_argument("--profile-directory=Default")
chrome_options.add_argument("--disable-plugins-discovery")
chrome_options.add_argument("--incognito")
chrome_options.add_argument("user_agent=DN")
self.browser = uc.Chrome(options=chrome_options)
self.browser.delete_all_cookies()
# example of loggin in to youtube without getting that issue
self.browser.get('http://youtube.com')
login_button_init = self.browser.find_element_by_xpath("//a[#aria-label='Sign in']")
login_button_init.click()
# locate the login button
login_button = self.browser.find_element_by_xpath("//paper-button[#aria-label='Sign in']")
login_button.click()
# get email and set to email input box
email = self.browser.find_element_by_id("identifierId")
myemail = os.environ.get('YOUTUBE_EMAIL')
email.send_keys(myemail)
# click next button
email_next_button = self.browser.find_element_by_id("identifierNext")
email_next_button.click()
# get password and set to password input box
password = self.browser.find_element_by_name("password")
mypassword = os.environ.get('YOUTUBE_PASSWORD')
password.send_keys(mypassword)
sleep(2)
# click next button to log in
pass_next_button = self.browser.find_element_by_id("passwordNext")
pass_next_button.click()
While navigating to the page, It is able to extract the data.
Code:
from selenium.webdriver.chrome.options import Options
options1 = Options()
options1.headless = True
driver = webdriver.Chrome(os.getcwd() +"/chromedriver",options = options1)
this is how I am getting the chrome driver
content = BeautifulSoup(driver.page_source,"html.parser")
this is how I am getting the content from the navigated page. While we are navigating to the page able to extract the data.
Can you try it like below?
from selenium import webdriver
options = webdriver.ChromeOptions()
options.add_argument('headless')
driver = webdriver.Chrome(os.getcwd() +"/chromedriver", chrome_options=options)