We Keep Coding

Pyhton for loop "stale element reference: element is not attached to the page document"

python for loop "stale element reference: element is not attached to the page document"
here is my code
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
import time
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('F:\\work\\chromedriver_win32\\chromedriver.exe')
driver = webdriver.Chrome(options=options, service=webdriver_service)
wait = WebDriverWait(driver, 10)
Content =["listtext1","listtext2","listtext3","listtext4"]
for i in range(4):
time.sleep(7)
url = "https://quillbot.com/"
driver.get(url)
Text_block = driver.find_element(By.ID, "inputText")
Text_block.send_keys(Content[i])# Change (fetch from Search_list)
time.sleep(2)

I made a few fixes to your code:
Inserted user_agent (it will come in handy with other experiments on selenium)
inserted a web driver manager to run selenium on all operating systems.
You have to accept cookies before you can interact with the page.
removed unnecessary sleep.
This is the result, code tested:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
options = Options()
options.add_argument("start-maximized")
# add user_agent
user_agent = "user-agent=[Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.63 Safari/537.36]"
options.add_argument(user_agent)
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options) # to use over all systems
browser_delay = 2 # set if based on your connection and device speed
Content =["listtext1","listtext2","listtext3","listtext4"]
for i in range(len(Content)):
url = "https://quillbot.com/"
driver.get(url)
try:
cookie_btn = WebDriverWait(driver, browser_delay).until(EC.element_to_be_clickable((By.ID, 'onetrust-accept-btn-handler')))
cookie_btn.click()
except TimeoutException:
pass # it's a timeout or element just clicked
Text_block = driver.find_element(By.ID, "inputText")
Text_block.send_keys(Content[i]) # Change (fetch from Search_list)

Here is one way of sending those texts into that textbox, based on your existing code and how you defined waits:
[....]
content =["listtext1","listtext2","listtext3","listtext4"]
for i in content:
driver.get('https://quillbot.com/')
try:
wait.until(EC.element_to_be_clickable((By.ID, "onetrust-accept-btn-handler"))).click()
print('accepted cookies')
except Exception as e:
print('no cookie button!')
text_block = wait.until(EC.element_to_be_clickable((By.ID, "inputText")))
text_block.send_keys(i)
print('sent', i)
t.sleep(5)
See Selenium documentation at https://www.selenium.dev/documentation/

Click on all elements of the same class at the same time

The code clicks on a element that may or may not exist on the page and then needs to click on all elements of the same class:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.firefox.options import Options
import time
my_user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36'
options = Options()
options.set_preference("general.useragent.override", my_user_agent)
options.page_load_strategy = 'eager'
options.add_argument('--headless')
driver = webdriver.Firefox(options=options)
driver.get("https://int.soccerway.com/matches/2022/07/23/")
try:
WebDriverWait(driver, 3).until(EC.presence_of_element_located((By.XPATH, "//a[contains(#class,'tbl-read-more-btn')]")))
driver.find_element(by=By.XPATH, value="//a[contains(#class,'tbl-read-more-btn')]").click()
time.sleep(0.1)
except:
pass
WebDriverWait(driver, 3).until(EC.presence_of_element_located((By.XPATH, "//tr[contains(#class,'group-head')]")))
for btn in driver.find_elements(by=By.XPATH, value="//tr[contains(#class,'group-head')]"):
btn.click()
time.sleep(0.1)
But this work takes 90 seconds to do and when I remove the time.sleep it drops to 65 seconds, but if I remove it I notice that in some very random times some of the elements that should be clicked are ignored.
Is there any way to do this same service but clicking all the elements at the same time to speed up the process?
Buttons to click on visual examples:
Expected Result after clicks (open the boxes):

In order to speed up the process you can click on all the competition-link one by one in a sequence using either of the following Locator Strategies:
Using CSS_SELECTOR:
for ele in driver.find_elements(By.CSS_SELECTOR, "tr.group-head.clickable th.competition-link>a"):
ele.click()
Using XPATH:
for ele in driver.find_elements(By.XPATH, "//tr[#class='group-head clickable ']//th[#class='competition-link']/a"):
ele.click()

Unable to locate element 'username' while logging in Instagram with Selenium hosted on Heroku

I have a problem with a headless mode of Selenium while making log in with Instagram. I have found a default web code snippet for Selenium headless mode, but it does not work to find a particular element on webpage (like username in Instagram main page). The code works fine locally on my pc, but when it is deployed on Heroku, it shows the error. The logs of error are below the code.
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
import random as rd
import os
import schedule
def job():
try:
user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36"
options = webdriver.ChromeOptions()
options.headless = True
options.add_argument(f'user-agent={user_agent}')
options.add_argument("--window-size=1920,1080")
options.add_argument('--ignore-certificate-errors')
options.add_argument('--allow-running-insecure-content')
options.add_argument("--disable-extensions")
options.add_argument("--proxy-server='direct://'")
options.add_argument("--proxy-bypass-list=*")
options.add_argument("--start-maximized")
options.add_argument('--disable-gpu')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--no-sandbox')
options.binary_location = os.environ.get("GOOGLE_CHROME_BINARY")
CHROMEDRIVER_PATH = os.environ.get("CHROMEDRIVER_PATH")
wd = webdriver.Chrome(executable_path=CHROMEDRIVER_PATH,chrome_options=options)
wd.get('https://instagram.com')
time.sleep(rd.uniform(9,11))
username = os.environ.get("INSTAGRAM_USER")
password = os.environ.get("INSTAGRAM_PASSWORD")
time.sleep(rd.uniform(2.5,3.5))
wd.find_element_by_name('username').send_keys(username)
time.sleep(rd.uniform(0.95,1.45))
wd.find_element_by_name('password').send_keys(password + Keys.ENTER)
time.sleep(rd.uniform(6,8))
wd.get('https://instagram.com')
time.sleep(rd.uniform(2.5,3.5))
print("SUCCESS")
wd.quit()
except Exception as e:
print(e)
schedule.every(3).minutes.do(job)
while True:
schedule.run_pending()
time.sleep(10)
Error:
Message: no such element: Unable to locate element: {"method":"css selector","selector":"[name="username"]"}
(Session info: headless chrome=88.0.4324.96)

To send a character sequence within the username field on Instagram you can use either of the following Locator Strategies:
Using css_selector:
driver.find_element(By.CSS_SELECTOR, "input[name='username']").send_keys("Artem")
Using xpath:
driver.find_element(By.XPATH, "//input[#name='username']").send_keys("Artem")
However, the username field on Instagram is a ReactJS enabled element, so ideally, to send a character sequence to the element you need to induce WebDriverWait for the element_to_be_clickable() and you can use either of the following Locator Strategies:
Using CSS_SELECTOR:
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "input[name='username']"))).send_keys("Artem")
Using XPATH:
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//input[#name='username']"))).send_keys("Artem")
Note: You have to add the following imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
Browser Snapshot:

Learning to scrape with Selenium and Python

I'm learning to scrape with selenium, but I'm having trouble connecting to this site 'http://www.festo.com/cat/it_it/products_VUVG_S?CurrentPartNo=8043720'
it does not load the content of the site
I would like to learn how to connect to this site to request images and data
my code is simple because I'm learning, I looked for ways to make the connection but without success
from selenium import webdriver
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
ff_profile = FirefoxProfile()
ff_profile.set_preference("general.useragent.override", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36")
driver = webdriver.Firefox(firefox_profile = ff_profile)
driver.get('http://www.festo.com/cat/it_it/products_VUVG_S?CurrentPartNo=8043720')
time.sleep(5)
campo_busca = driver.find_elements_by_id('of132')
print(campo_busca)

As the the desired element is within an <iframe> so to invoke extract the src attribute of the desired element you have to:
Induce WebDriverWait for the desired frame to be available and switch to it.
Induce WebDriverWait for the desired visibility_of_element_located().
You can use the following Locator Strategies:
driver = webdriver.Firefox(executable_path=r'C:\Utility\BrowserDrivers\geckodriver.exe')
driver.get('http://www.festo.com/cat/it_it/products_VUVG_S?CurrentPartNo=8043720')
WebDriverWait(driver, 20).until(EC.frame_to_be_available_and_switch_to_it((By.XPATH,"//iframe[#id='CamosIFId' and #name='CamosIF']")))
print(WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.XPATH, "//img[#id='of132']"))).get_attribute("src"))
However as in one of the comments #google mentioned, it seems the browsing experiance is better with ChromeDriver / Chrome and you can use the following solution:
options = webdriver.ChromeOptions()
options.add_argument("start-maximized")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
driver = webdriver.Chrome(options=options, executable_path=r'C:\Utility\BrowserDrivers\chromedriver.exe')
driver.get('http://www.festo.com/cat/it_it/products_VUVG_S?CurrentPartNo=8043720')
WWebDriverWait(driver, 20).until(EC.frame_to_be_available_and_switch_to_it((By.CSS_SELECTOR,"iframe#CamosIFId[name='CamosIF']")))
print(WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.CSS_SELECTOR, "img#of132"))).get_attribute("src"))
Note : You have to add the following imports :
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
Console Output:
https://www.festo.com/cfp/camosHtml/i?SIG=0020e295a546f45d9acb6844231fd8ff31ca817a_64_64.png
Here you can find a relevant discussion on Ways to deal with #document under iframe

try this
for more information here
FIREFOX_DRIVER_PATH = "your_geckodriver_path"
firefox_options = FirefoxOptions()
firefox_options.headless = True
# set options as per requirement for firefox
firefox_options.add_argument("--no-sandbox")
firefox_options.add_argument("--disable-setuid-sandbox")
firefox_options.add_argument('--disable-dev-shm-usage')
firefox_options.add_argument("--window-size=1920,1080")
driver = webdriver.Firefox(firefox_options=firefox_options, executable_path=FIREFOX_DRIVER_PATH)
driver.get('http://www.festo.com/cat/it_it/products_VUVG_SCurrentPartNo=8043720')
time.sleep(5)
campo_busca = driver.find_elements_by_id('of132')
print(campo_busca)

download the driver from this link
and place it a folder and copy the complete path and paste below
FIREFOX_DRIVER_PATH = "driver_path"
firefox_options = FirefoxOptions()
#only if you dont want to see the gui else make is false or comment
firefox_options.headless = True
driver = webdriver.Firefox(firefox_options=firefox_options, executable_path=FIREFOX_DRIVER_PATH)
driver.get('http://www.festo.com/cat/it_it/products_VUVG_SCurrentPartNo=8043720')
time.sleep(3)
campo_busca = driver.find_elements_by_id('of132')
print(campo_busca)

How to click youtube link within youtube comment using an user agent through Selenium and Python

I'm learning about python selenium, I want to click a youtube link in the youtube comment, can someone help me?
Example: URL
Html :
<a class="yt-simple-endpoint style-scope yt-formatted-string" spellcheck="false" href="/watch?v=PbLtyVcMrk0">https://www.youtube.com/watch?v=PbLtyVcMrk0</a>
Code trials :
from selenium import webdriver
from fake_useragent import UserAgent
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
ua = UserAgent()
options = webdriver.ChromeOptions()
userAgent = ua.random
print(userAgent)
options.add_argument('user-agent={userAgent}')
driver = webdriver.Chrome(chrome_options=options)
driver.get("https://www.youtube.com/watch?v=NIWwJbo-9_8&lc=UgwNBxYVXb6uiVTioPB4AaABAg")
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//a[#class='yt-uix-sessionlink ' and contains(#href, '/watch?v=PbLtyVcMrk0')]"))).click()

You were pretty close. To click on the desired comment with text as https://www.youtube.com/watch?v=PbLtyVcMrk0 within the url you need to induce WebDriverWait for the element to be clickable and you can use the following solution using useragent through Selenium and Python:
Code Block:
from selenium import webdriver
from fake_useragent import UserAgent
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
ua = UserAgent()
options = webdriver.ChromeOptions()
userAgent = ua.random
print(userAgent)
options.add_argument('user-agent=' + userAgent)
driver = webdriver.Chrome(chrome_options=options)
driver.get("https://www.youtube.com/watch?v=NIWwJbo-9_8&lc=UgwNBxYVXb6uiVTioPB4AaABAg")
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//a[#class='yt-uix-sessionlink spf-link ' and contains(#href, '/watch?v=PbLtyVcMrk0')]"))).click()
Console Output:
Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36

The problem is with your xpath, also the logic which you implemented to perform click operation could be more refined like this:
from selenium import webdriver
from fake_useragent import UserAgent
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
ua = UserAgent()
options = webdriver.ChromeOptions()
userAgent = ua.random
print(userAgent)
options.add_argument('user-agent={userAgent}')
driver = webdriver.Chrome(chrome_options=options)
driver.get("https://www.youtube.com/watch?v=NIWwJbo-9_8&lc=UgwNBxYVXb6uiVTioPB4AaABAg")
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "yt-formatted-string[class*='ytd-comment-renderer'][id='content-text']>a")))
clickLinks = driver.find_elements_by_css_selector("yt-formatted-string[class*='ytd-comment-renderer'][id='content-text']>a")
for element in clickLinks:
if 'youtube' in element.text:
element.click()
Hope this helps.