We Keep Coding

Problem with "Web Driver Wait" in Selenium: By.CSS_SELECTOR button

This is a script, for my personal study and my educational/didactic purpose, with Tor and Selenium connection.
Both the scraping (team name list) and the Tor connection worked fine.
Then I added a code with Web Driver Wait to press the Cookie button, but now nothing works correctly anymore. The code entered is in contrast to that of Tor.
How can I solve by keeping both the Tor code and the Web Driver Wait code active?
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
import sqlite3
### CONNESSIONE TOR ###
from selenium import webdriver
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
import os
torexe = os.popen('/home/mypc/.local/share/torbrowser/tbb/x86_64/tor-browser_en-US')
profile = FirefoxProfile('/home/mypc/.local/share/torbrowser/tbb/x86_64/tor-browser_en-US/Browser/TorBrowser/Data/Browser/profile.default')
profile.set_preference('network.proxy.type', 1)
profile.set_preference('network.proxy.socks', '127.0.0.1')
profile.set_preference('network.proxy.socks_port', 9050)
profile.set_preference("network.proxy.socks_remote_dns", False)
profile.update_preferences()
firefox_options = webdriver.FirefoxOptions()
firefox_options.binary_location = '/usr/bin/firefox'
driver = webdriver.Firefox(
firefox_profile=profile, options=firefox_options,
executable_path='/usr/bin/geckodriver')
#Scraping SerieA
driver.maximize_window()
wait = WebDriverWait(driver, 20)
driver.get("link")
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button[id='onetrust-accept-btn-handler']"))).click()
for SerieA in driver.find_elements(By.CSS_SELECTOR, "a[href^='/squadra'][class^='rowCellParticipantName']"):
print(SerieA.text)
### SALVARE IN DATABASE: Nomi Campionati
con = sqlite3.connect('/home/mypc/Scrivania/folder/Database.db')
cursor = con.cursor()
records_added_Risultati = 0
Values = SerieA
sqlite_insert_query = 'INSERT INTO ARCHIVIO_Squadre_Campionato (Nome_Squadra) VALUES (?);'
count = cursor.executemany(sqlite_insert_query, Values) #executemany, no execute
con.commit()
print("Record inserted successfully ", cursor.rowcount)
records_added_Risultati = records_added_Risultati + 1
cursor.close()
The error is:
Traceback (most recent call last):
File "/usr/lib/python3.8/idlelib/run.py", line 559, in runcode
exec(code, self.locals)
File "/home/mypc/Scrivania/folder/example.py", line 31, in <module>
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button[id='onetrust-accept-btn-handler']"))).click()
File "/home/mypc/.local/lib/python3.8/site-packages/selenium/webdriver/support/wait.py", line 80, in until
raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message:

You are mixing implicitly wait with explicit wait. This is not recommended and causes problems.
I'm quite sure your problems are caused by this.
You can read more about this here
Also see this
UPD
You have a typo in both element locators.
Just a spaces, but totally braking the locators.
Also, in case accept cookies is not stable i.e. sometimes not appearing put it inside try-accept block as following:
driver.maximize_window()
wait = WebDriverWait(driver, 10)
driver.get("https://www.thesiteurl/bla/bla/discrete")
try:
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button[id='onetrust-accept-btn-handler']"))).click()
except:
pass
for SerieA in driver.find_elements(By.CSS_SELECTOR, "a[href^='/squadra'][class^='rowCellParticipantName']"):
print(SerieA.text)

Page refreshing after pressing button in selenium?

Hi i have this code with tries many number on yahoo after the button of submitting is clicked
the page seems to get refreshed because i am not able to press the same button.
here is my code
import os
from tkinter import *
from tkinter import filedialog
import selenium
import time
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium import webdriver
from selenium import webdriver
import contextlib as textmanager
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
PATH= "C:\chromedrivers\chromedriver.exe"
driver= webdriver.Chrome(PATH)
list=[]
file= open("numbers.txt", "r")
for line in file:
line = line.strip() #preprocess line
list.append(line)
driver.get("https://www.yahoo.com/")
element = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//*[#id='consent-page']/div/div/div/form/div[2]/div[2]/button")))
element.click();
time.sleep(1)
sign_in= driver.find_element_by_xpath("//*[#id='ybar-inner-wrap']/div[3]/div/div[3]/div[1]/div/a").click()
time.sleep(1)
forgot_phone= driver.find_element_by_xpath("//*[#id='mbr-forgot-link']").click()
time.sleep(2)
inputt= driver.find_element_by_id("username")
time.sleep(2)
buttonn= driver.find_element_by_xpath("//*[#id='yid-challenge']/form/div[2]/button")
counter=0
nonumber= open("nonumber.txt","w")
while counter != len(list):
inputt.send_keys(list[counter])
time.sleep(1)
buttonn.click()
if len(driver.find_elements_by_css_selector("p[data-error]")) > 0 :
inputt.clear()
nonumber.write(list[counter])
nonumber.close()
counter=counter+1
here is the error it says button not clickable but in first run it works iam not sure is the problem in button or input area
Traceback (most recent call last):
File "c:\Users\ramhelsinki\projects\slenium.py", line 54, in <module>
inputt.send_keys(list[counter])
File "C:\Users\ramhelsinki\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\selenium\webdriver\remote\webelement.py", line 477, in send_keys
self._execute(Command.SEND_KEYS_TO_ELEMENT,
File "C:\Users\ramhelsinki\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\selenium\webdriver\remote\webelement.py", line 633, in _execute
return self._parent.execute(command, params)
File "C:\Users\ramhelsinki\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\selenium\webdriver\remote\webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "C:\Users\ramhelsinki\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.StaleElementReferenceException: Message: stale element reference: element is not attached to the page document
(Session info: chrome=91.0.4472.124)

Each time the web page is changed / refreshed all the web elements previously fetched by Selenium become irrelevant. This is what stale element reference exception comes for.
With your code, since page is refreshed in case of wrong input, I would advice fetching the elements again after each time the page is changed so the code will look like this:
time.sleep(2)
counter=0
nonumber= open("nonumber.txt","w")
while counter != len(list):
inputt= driver.find_element_by_id("username")
buttonn= driver.find_element_by_xpath("//button")
inputt.send_keys(list[counter])
time.sleep(1)
buttonn.click()
if len(driver.find_elements_by_css_selector("p[data-error]")) > 0 :
inputt= driver.find_element_by_id("username")
inputt.clear()
nonumber.write(list[counter])
nonumber.close()
counter=counter+1

Python selenium “while true:” script can't execute driver.get(“URL”)

My script has a while True: script running 24/7. It always hits print ('False') every sleep(55) seconds, and once the current_time is between specified times, it calls driver.get("URL") and does a few actions on a website.
But sometimes it hits an error where it cannot open driver.get('https://www.instagram.com/accounts/login/?source=auth_switcher') and prints the error below.
Traceback (most recent call last):
File "/home/matt/insta/users/nycforest/lcf.py", line 254, in <module>
lcf_time(input_begin_time,input_end_time,input_begin_time2,input_end_time2)
File "/home/matt/insta/users/nycforest/lcf.py", line 241, in lcf_time
login()
File "/home/matt/insta/users/nycforest/lcf.py", line 40, in login
driver.get('https://www.instagram.com/accounts/login/?source=auth_switcher')
File "/home/matt/.local/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 333, in get
self.execute(Command.GET, {'url': url})
File "/home/matt/.local/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "/home/matt/.local/lib/python3.8/site-packages/selenium/webdriver/remote/errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message: timeout
(Session info: headless chrome=78.0.3904.70)
(Driver info: chromedriver=2.42.591071 (0b695ff80972cc1a65a5cd643186d2ae582cd4ac),platform=Linux 5.4.0-1020-aws x86_64)
I tried adding driver.set_page_load_timeout(20) after driver.get('URL') but it hits the error above
from pyvirtualdisplay import Display
from datetime import datetime, time
from itertools import islice
from random import randint
from time import sleep
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException
input_begin_time = time(18,40)
input_end_time = time(18,50)
input_begin_time2 = time(21,20)
input_end_time2 = time(21,30)
opts = webdriver.ChromeOptions()
opts.add_argument('--headless')
opts.add_argument('--disable-gpu')
opts.add_argument('--no-sandbox')
opts.add_argument('--disable-dev-shm-usage')
opts.add_argument('--enable-features=NetworkService,NetworkServiceInProcess')
def login():
sleep(2)
driver.get('https://www.instagram.com/accounts/login/?source=auth_switcher')
driver.set_page_load_timeout(20)
sleep(3)
input_username = driver.find_element_by_name('username').send_keys(username)
input_password = driver.find_element_by_name('password').send_keys(password)
button_login = driver.find_element_by_css_selector('#loginForm > div > div:nth-child(3) > button')
button_login.click()
# def other_functions()
driver = webdriver.Chrome(options=opts)
def lcf_time(time_begin1, time_end1, time_begin2, time_end2, curren_time=None):
current_time = datetime.now().time()
if time_begin1 < current_time < time_end1 or time_begin2 < current_time < time_end2:
login()
# other_functions()
driver.close()
else:
print("False")
while True:
lcf_time(input_begin_time,input_end_time,input_begin_time2,input_end_time2)
sleep(55)
The odd thing is: I have a few different scripts running the exact same script with different variables and it some scripts are fine and others consistently hit this 60% of the time. I need a more reliable way to driver.get("URL") 24/7
Is this a compatibility issue? Am I doing something wrong?

Maximum Recursion Error with Selenium's driver.get() call

I am using selenium to running automated scripts in Python. I have used the tool for three years, but have never encountered this issue. Does anyone know what could be causing this? I was able to determine that the cause of the error was the reference to driver.get() inside a for loop, but it errors out after 7 iterations. Seems odds, thoughts?
Unhandled exception in thread started by <function crawl_games_and_store_data.<locals>.handle_incoming_request at 0x104659158>
Traceback (most recent call last):
File "/Users/z003bzf/Documents/personal/python/MLB/src/services/crawler.py", line 160, in handle_incoming_request
driver.get(game_link)
File "/Users/z003bzf/.local/share/virtualenvs/MLB-Ei2Ym8vD/lib/python3.7/site-packages/selenium/webdriver/remote/webdriver.py", line 333, in get
self.execute(Command.GET, {'url': url})
File "/Users/z003bzf/.local/share/virtualenvs/MLB-Ei2Ym8vD/lib/python3.7/site-packages/selenium/webdriver/remote/webdriver.py", line 318, in execute
params = self._wrap_value(params)
File "/Users/z003bzf/.local/share/virtualenvs/MLB-Ei2Ym8vD/lib/python3.7/site-packages/selenium/webdriver/remote/webdriver.py", line 271, in _wrap_value
converted[key] = self._wrap_value(val)
File "/Users/z003bzf/.local/share/virtualenvs/MLB-Ei2Ym8vD/lib/python3.7/site-packages/selenium/webdriver/remote/webdriver.py", line 276, in _wrap_value
return list(self._wrap_value(item) for item in value)
Here is the code that is causing the issue
for elem in link_lst:
driver.get(elem)
time.sleep(.5)
driver.find_element_by_xpath('//div[#class="box-row batting-row"]')

It could be the contents of link_lst, if there's a timeout on one of the hosts. You would have to handle this exception to continue forward. One possible option would be to use a try/except on a timeout as well as not being able to locate a page element. This can be adjusted both as a delay parameter, but also in the firefox profile settings.
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
def fetch(driver, link_list, delay)
for item in link_list:
try:
driver.get(item)
except TimeoutException:
print("Timeout in link_list item")
try:
elem = WebDriverWait(driver,
delay).until(EC.presence_of_element_located((By.XPATH '//div[#class="box-row batting-row"]')))
except TimeoutException:
print("Locating element took too much time!")
if __name__ == '__main__':
""" Setup profile """
fp = webdriver.FirefoxProfile()
fp.set_preference("http.response.timeout", 10)
fp.set_preference("dom.max_script_run_time", 10)
""" Prepare args """
driver = webdriver.Firefox(firefox_profile=fp)
link_list = ['http://abcdeftest.com', 'http://test.com']
delay = 5
fetch(driver, link_list, delay)

python and selenium send keys

Im trying to add input to a text box when i do try it doesn't find the element and it gives an error i don't know if im selecting the right element this is what i have so far
Traceback (most recent call last):
File "./fl_bot.py", line 22, in <module>
ui.WebDriverWait(browser, 10).until(EC.visibility_of_element_located((By.ID, "#billFirstName")))
File "/Library/Python/2.7/site-packages/selenium/webdriver/support/wait.py", line 80, in until
raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message:
the code abouve is the error message im getting
from selenium.webdriver.support import ui
from selenium import webdriver
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
def get_page(model, sku):
url = "https://www.footlocker.com/product/model:"+str(model)+"/sku:"+ str(sku)+"/"
return url
browser = webdriver.Firefox()
page=browser.get(get_page(277097,"8448001"))
browser.find_element_by_xpath("//*[#id='pdp_size_select_mask']").click()
shoesize = ui.WebDriverWait(browser, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, 'a.grid_size:nth-child(8)')))
shoesize.click()
browser.find_element_by_xpath("//*[#id='pdp_addtocart_button']").click()
checkout = browser.get('https://www.footlocker.com/shoppingcart/default.cfm?sku=')
checkoutbutton = browser.find_element_by_css_selector('#cart_checkout_button').click()
ui.WebDriverWait(browser, 10).until(EC.visibility_of_element_located((By.ID, "#billFirstName")))
browser.find_element_by_id("#billFirstName").send_keys(Keys.RETURN)
every time it makes it to the end of the script it dosent type and it just stops
[1]: https://www.footlocker.com/checkout/?uri=checkout this is the page im trying to check out on

The FIRST NAME is a mandatory field to be filled up, so instead of send_keys(Keys.RETURN) try to send some text as follows along with the expected_condition as element_to_be_clickable :
ui.WebDriverWait(browser, 10).until(EC.element_to_be_clickable((By.XPATH, "//input[#id='billFirstName']")))
browser.find_element_by_xpath("//input[#id='billFirstName']").click()
browser.find_element_by_xpath("//input[#id='billFirstName']").clear()
browser.find_element_by_xpath("//input[#id='billFirstName']").send_keys("user_first_name")