We Keep Coding

Selenium / Try to click button in shadow-root?

i try to click the cookie all accept button on this page:
https://www.tiktok.com/#shneorgru
with the following code
import time
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from sys import platform
import os, sys
import xlwings as xw
from datetime import datetime, timedelta
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager
from fake_useragent import UserAgent
if __name__ == '__main__':
SAVE_INTERVAL = 5
WAIT = 1
print(f"Checking chromedriver...")
os.environ['WDM_LOG_LEVEL'] = '0'
ua = UserAgent()
userAgent = ua.random
options = Options()
# options.add_argument('--headless')
options.add_argument("start-maximized")
options.add_experimental_option("prefs", {"profile.default_content_setting_values.notifications": 1})
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('excludeSwitches', ['enable-logging'])
options.add_experimental_option('useAutomationExtension', False)
options.add_argument('--disable-blink-features=AutomationControlled')
options.add_argument(f'user-agent={userAgent}')
srv=Service(ChromeDriverManager().install())
driver = webdriver.Chrome (service=srv, options=options)
waitWebDriver = WebDriverWait (driver, 10)
link = f"https://www.tiktok.com/#shneorgru"
driver.get (link)
time.sleep(WAIT)
driverElem = driver.find_element(By.XPATH,"//tiktok-cookie-banner")
root1 = driver.execute_script("return arguments[0].shadowRoot", driverElem)
root1.find_element(By.XPATH,"(//button)[2]").click()
time.sleep(WAIT)
I try to select first the tag which is around the shadwo-root.
Then try to execute the script for the shadowRoot.
And then find the element inside the shadowRoot and finally click the button.
But i allways get the following error:
$ python collTikTok.py
Checking chromedriver...
Traceback (most recent call last):
File "C:\Users\Polzi\Documents\DEV\Fiverr\TRY\rosen771\collTikTok.py", line 56, in <module>
root1.find_element(By.XPATH,"(//button)[2]").click()
File "C:\Users\Polzi\Documents\DEV\.venv\selenium\lib\site-packages\selenium\webdriver\remote\shadowroot.py", line 59, in find_element
return self._execute(
File "C:\Users\Polzi\Documents\DEV\.venv\selenium\lib\site-packages\selenium\webdriver\remote\shadowroot.py", line 94, in _execute
return self.session.execute(command, params)
File "C:\Users\Polzi\Documents\DEV\.venv\selenium\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 425, in execute
self.error_handler.check_response(response)
File "C:\Users\Polzi\Documents\DEV\.venv\selenium\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 247, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.InvalidArgumentException: Message: invalid argument: invalid locator
How can i click this cookie accept button?
Here is the code how it looks like in the shadow-root:

Page refreshing after pressing button in selenium?

Hi i have this code with tries many number on yahoo after the button of submitting is clicked
the page seems to get refreshed because i am not able to press the same button.
here is my code
import os
from tkinter import *
from tkinter import filedialog
import selenium
import time
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium import webdriver
from selenium import webdriver
import contextlib as textmanager
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
PATH= "C:\chromedrivers\chromedriver.exe"
driver= webdriver.Chrome(PATH)
list=[]
file= open("numbers.txt", "r")
for line in file:
line = line.strip() #preprocess line
list.append(line)
driver.get("https://www.yahoo.com/")
element = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//*[#id='consent-page']/div/div/div/form/div[2]/div[2]/button")))
element.click();
time.sleep(1)
sign_in= driver.find_element_by_xpath("//*[#id='ybar-inner-wrap']/div[3]/div/div[3]/div[1]/div/a").click()
time.sleep(1)
forgot_phone= driver.find_element_by_xpath("//*[#id='mbr-forgot-link']").click()
time.sleep(2)
inputt= driver.find_element_by_id("username")
time.sleep(2)
buttonn= driver.find_element_by_xpath("//*[#id='yid-challenge']/form/div[2]/button")
counter=0
nonumber= open("nonumber.txt","w")
while counter != len(list):
inputt.send_keys(list[counter])
time.sleep(1)
buttonn.click()
if len(driver.find_elements_by_css_selector("p[data-error]")) > 0 :
inputt.clear()
nonumber.write(list[counter])
nonumber.close()
counter=counter+1
here is the error it says button not clickable but in first run it works iam not sure is the problem in button or input area
Traceback (most recent call last):
File "c:\Users\ramhelsinki\projects\slenium.py", line 54, in <module>
inputt.send_keys(list[counter])
File "C:\Users\ramhelsinki\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\selenium\webdriver\remote\webelement.py", line 477, in send_keys
self._execute(Command.SEND_KEYS_TO_ELEMENT,
File "C:\Users\ramhelsinki\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\selenium\webdriver\remote\webelement.py", line 633, in _execute
return self._parent.execute(command, params)
File "C:\Users\ramhelsinki\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\selenium\webdriver\remote\webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "C:\Users\ramhelsinki\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.StaleElementReferenceException: Message: stale element reference: element is not attached to the page document
(Session info: chrome=91.0.4472.124)

Each time the web page is changed / refreshed all the web elements previously fetched by Selenium become irrelevant. This is what stale element reference exception comes for.
With your code, since page is refreshed in case of wrong input, I would advice fetching the elements again after each time the page is changed so the code will look like this:
time.sleep(2)
counter=0
nonumber= open("nonumber.txt","w")
while counter != len(list):
inputt= driver.find_element_by_id("username")
buttonn= driver.find_element_by_xpath("//button")
inputt.send_keys(list[counter])
time.sleep(1)
buttonn.click()
if len(driver.find_elements_by_css_selector("p[data-error]")) > 0 :
inputt= driver.find_element_by_id("username")
inputt.clear()
nonumber.write(list[counter])
nonumber.close()
counter=counter+1

Python Selenium WebDriver unable to find element at all

Selenium webdriver has been unable to find any elements on page using different methods: class_name, id, & xpath.
Here's my code:
from selenium import webdriver
##from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.keys import Keys
import time
import random
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--disable-gpu")
driver = webdriver.Chrome(executable_path=r'C:\Users\acer\Downloads\chromedriver_win32\chromedriver.exe', chrome_options=chrome_options)
time.sleep(2)
driver.get('https://www.reddit.com/r/AskReddit/comments/fi04fh/what_are_some_spoilers_for_the_next_month_of_2020/')
time.sleep(2)
print(driver.title)
time.sleep(2)
element = driver.find_element_by_id("header")
print("done")
The title prints successfully but it fails on the line of driver.find_element_by_id("header").
In fact, I am trying to find the element whose class is "top-matter" (using find_by_class_name) but since this wasn't working, I tested it for other elements ("header") using respective methods ("xpath", "id") but nothing is working for me.
Can anyone provide some insight into the issue?
EDIT: Here's the error:
Traceback (most recent call last):
File "C:/Python34/reddit_test.py", line 20, in <module>
element = driver.find_element_by_id("header")
File "C:\Python34\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 269, in find_element_by_id
return self.find_element(by=By.ID, value=id_)
File "C:\Python34\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 752, in find_element
'value': value})['value']
File "C:\Python34\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 236, in execute
self.error_handler.check_response(response)
File "C:\Python34\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 192, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"id","selector":"header"}
(Session info: headless chrome=80.0.3987.132)
(Driver info: chromedriver=2.41.578737 (49da6702b16031c40d63e5618de03a32ff6c197e),platform=Windows NT 6.1.7601 SP1 x86_64)
Here's proof that the element exists...

in your url there is no header id
to ignore this exception
try this code:
from selenium import webdriver
##from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.keys import Keys
import time
import random
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--disable-gpu")
driver = webdriver.Chrome(executable_path=r'C:\Users\acer\Downloads\chromedriver_win32\chromedriver.exe', chrome_options=chrome_options)
time.sleep(2)
driver.get('https://www.reddit.com/r/AskReddit/comments/fi04fh/what_are_some_spoilers_for_the_next_month_of_2020/')
time.sleep(2)
print(driver.title)
time.sleep(2)
try:
element = driver.find_element_by_id("header")
except:
print("The Header isd dose not exist!")
exit()
print("done")
In Your Url The Header is dose not exist
You Can See By This Image!

The issue was that I use the old version of reddit whereas the default which Selenium and Hamza were opening is the new version which doesn't contain the elements I was trying to find

Python selenium refresh if wait more than 10s

from selenium import webdriver
driver=webdriver.Firefox()
driver.get(url)
Sometimes the webdriver is stuck on a file or response and the page is never full-loaded so the line
driver.get(url)
is never finished. But I already got enough source code to run the rest of my code. I am wondering how can I bypass or refresh the page if the page is not full-loaded in 10 seconds.
I have tried
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
driver=webdriver.Firefox()
driver.set_page_load_timeout(10)
while True:
try:
driver.get(url)
except TimeoutException:
print("Timeout, retrying...")
continue
else:
break
but the line
driver.set_page_load_timeout(10)
always gives me following error
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/selenium/webdriver/remote/webdriver.py", line 727, in set_page_load_timeout
'pageLoad': int(float(time_to_wait) * 1000)})
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/selenium/webdriver/remote/webdriver.py", line 238, in execute
self.error_handler.check_response(response)
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/selenium/webdriver/remote/errorhandler.py", line 193, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.WebDriverException: Message:
This is nothing after Message:. I can't identify the type of error. It's weird that my laptop can't run
driver.set_page_load_timeout(10)
My next step is to click a button on the page, but that button doesn't always exist even after full-loaded. Thus I can't use explicit wait.
Thanks

(In your code snippet you don't define URL, but I'll assume URL is defined somewhere in your actual code.)
You could combine the retry and timeout-decorator packages for this:
from retry import retry
from timeout_decorator import timeout, TimeoutError
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
#retry(TimeoutError, tries=3)
#timeout(10)
def get_with_retry(driver, url):
driver.get(url)
def main():
url = "http://something.foo"
driver=webdriver.Firefox()
try:
get_with_retry(driver, url)
foo(driver) # do whatever it is you need to do
finally:
driver.quit()
if __name__ == "__main__":
main()
Note that you would need to either not set driver.set_page_load_timeout to anything, or set it to something higher than 10 seconds.

Selenium random timeout exceptions without any message

Here is what I'm trying to do and most of the time I succeed:
Basically I'm signing in on a website and then wait for a class to be in the source, then process the source code.
The exception I get:
Traceback (most recent call last):
File "foo.py", line 495, in <module>
report(login, password)
File "foo.py", line 430, in report
data = bar(login, password)
File "foo.py", line 113, in
ui.WebDriverWait(browser, 10).until(lambda browser: browser.find_elements_by_class_name("the-class-i-want"))
File "/Library/Python/2.7/site-packages/selenium/webdriver/support/wait.py", line 71, in until
raise TimeoutException(message)
selenium.common.exceptions.TimeoutException: Message: ''
Here is the code:
from selenium import webdriver
import contextlib
from selenium.webdriver.common.keys import Keys
import selenium.webdriver.support.ui as ui
from selenium.webdriver.support.wait import WebDriverWait
with contextlib.closing(webdriver.PhantomJS('phantomjs')) as browser:
browser.get('mywebsite')
login_form = browser.find_element_by_id('login-form')
email = browser.find_element_by_name('login')
password = browser.find_element_by_name('password')
email.send_keys(login)
password.send_keys(password)
password.send_keys(Keys.RETURN)
ui.WebDriverWait(browser, 10).until(lambda browser: browser.find_elements_by_class_name("the-class-i-want"))
I tried this too:
wait_count = 0
while wait_count < 6:
print wait_count
ui.WebDriverWait(browser, 10).until(lambda browser: browser.find_elements_by_class_name("the-class-i-want"))
if browser.find_elements_by_class_name("the-class-i-want"):
break
wait_count += 1
I get the same exception.
I am currently trying this :
wait_count = 0
while wait_count < 6:
try:
ui.WebDriverWait(browser, 10).until(lambda browser: browser.find_elements_by_class_name("the-class-i-want"))
if browser.find_elements_by_class_name("the-class-i-want"):
break
except:
wait_count += 1
continue
I haven't got to the point where it fails, I'm still testing it.
Sorry this is very long. But I'd like to find a pythonic and clean solution to those random timeouts.
Another info that could help too: the signing in process is sometimes very long, but even with a several minutes wait, it throws the exception.

Here is the answer I got after contacting Adam Goucher:
from selenium import webdriver
import contextlib
from selenium.webdriver.common.keys import Keys
import selenium.webdriver.support.ui as ui
from selenium.webdriver.support.wait import WebDriverWait
def waiter(browser):
elements = browser.find_elements_by_class_name('the-class-i-want')
if len(elements) != 0:
return elements
return False
with contextlib.closing(webdriver.PhantomJS('phantomjs')) as browser:
browser.get('mywebsite')
login_form = browser.find_element_by_id('login-form')
email = browser.find_element_by_name('login')
password = browser.find_element_by_name('password')
email.send_keys(login)
password.send_keys(password)
password.send_keys(Keys.RETURN)
ui.WebDriverWait(browser, 10).until(waiter)
And this works perfectly fine!