We Keep Coding

Selenium, how to locate and click a particular button

I'm using selenium to try and scrape a listing of products in this website:
https://www.zonacriativa.com.br/harry-potter
However, I'm having trouble getting the full listing of products. the page list 116 products, yet only a few are shown at a time. If I want to see the other ones, I need to click on the "Carregar mais Produtos" (load more products) button at the bottom a few times to get the full listing.
I'm having trouble locating this button, as it doesn't have an id and its class is a huge string. I've tried several things, like the examples below, but they don't seem to work. Any suggestions?
driver.find_element("xpath", "//button[text()='Carregar mais Produtos']").click()
driver.find_element("css selector", ".vtex-button__label.flex.items-center.justify-center.h-100.ph5").click()
driver.find_element(By.CLASS_NAME, "vtex-button.bw1.ba.fw5.v-mid.relative.pa0.lh-solid.br2.min-h-small.t-action--small.bg-action-primary.b--action-primary.c-on-action-primary.hover-bg-action-primary.hover-b--action-primary.hover-c-on-action-primary.pointer").click()

The element you trying to click is initially out of the visible screen so you can't click it. Also this XPath at least for me doesn't locate that element.
What you need to do is to scroll the page down untill that button becomes visible and clickable and then click it.
The following code clicks that button 1 time:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(options=options, service=webdriver_service)
wait = WebDriverWait(driver, 5)
url = "https://www.zonacriativa.com.br/harry-potter"
driver.get(url)
while True:
try:
wait.until(EC.element_to_be_clickable((By.XPATH, "//div[contains(#class,'buttonShowMore')]//button"))).click()
break
except:
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
The above code can be simply modified to scroll and click that button until we reach the latest page where this button is not presented:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(options=options, service=webdriver_service)
wait = WebDriverWait(driver, 5)
url = "https://www.zonacriativa.com.br/harry-potter"
driver.get(url)
while driver.find_elements(By.XPATH, "//div[contains(#class,'buttonShowMore')]//button"):
try:
wait.until(EC.element_to_be_clickable((By.XPATH, "//div[contains(#class,'buttonShowMore')]//button"))).click()
except:
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

How can I use selenium to click accept all cookies on the website "sky news"

enter image description here
I want to make a web scraper for the news title on the news website, news.sky.com
The problem is, very often, the site will pop up a message to let me "accept all cookies" (see capture...)
I followed the guide on the similar question by using XPath. But, it feedbacks:
raise TimeoutException(message, screen, stacktrace)
Seemly, selenium can't find out the location based on XPath.
so, is it possible to click "Accept all"?
please help, thx.
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium import webdriver
from selenium.webdriver.edge.service import Service
from selenium.webdriver.common.by import By
e_driver_path = r"F:/Download/Portable Test/msedgedriver.exe"
# establish the web driver
s = Service(e_driver_path)
driver = webdriver.Edge(service=s)
driver.get("https://news.sky.com/uk")
# search = driver.find_element_by_id(ContentPlaceHolder1_NotifyBtn)
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//*[#id='notice']/div[3]/button[1]"))).click()
print("yes")

The cookie buttons are in an iframe on that page. Here is one way of clicking that button:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.keys import Keys
import time as t
chrome_options = Options()
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument('disable-notifications')
chrome_options.add_argument("window-size=1280,720")
webdriver_service = Service("chromedriver/chromedriver") ## path to where you saved chromedriver binary
driver = webdriver.Chrome(service=webdriver_service, options=chrome_options)
actions = ActionChains(driver)
wait = WebDriverWait(driver, 20)
url = "https://news.sky.com/uk"
driver.get(url)
wait.until(EC.frame_to_be_available_and_switch_to_it((By.XPATH, "//iframe[#title='SP Consent Message']")))
try:
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button[title='Accept all']"))).click()
print('accepted cookies')
except Exception as e:
print('no cookie button!')
driver.switch_to.default_content()
print('back to main content')
This will access the iframe, click the button, and then exit the iframe back to main content, and also print in terminal:
accepted cookies
back to main content
Selenium setup is chrome/chromedriver/linux, you just need to observe the import and the code after defining the driver, to adapt it to your own setup. Selenium docs can be found here: https://www.selenium.dev/documentation/

Selenium :Cant find and click element by X-path

I am trying to click a button on website using selenium. But selenium cant find it. It is not iframe or problem with X-path. What I think is there is some event to make the element available by javascript. I tried to get it using class although it had very bad class value. If any can help. It will be a relief for me. I tried my best. I got the X-path from firefox developer tools.The url of site is :https://www.bedbathandbeyond.com/store/category/kitchen/trash-recycling/14367
I want to click the "next" button (available almost in the bottom)
from selenium import webdriver
from selenium.webdriver.chrome import service
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
option = webdriver.ChromeOptions()
option.add_argument("start-maximized")
option.add_experimental_option("excludeSwitches", ["enable-automation"])
option.add_experimental_option('useAutomationExtension', False)
option.add_argument("--disable-blink-features")
option.add_argument("--disable-gpu")
#option.add_argument("--headless")
option.add_argument('--disable-blink-features=AutomationControlled')
wd = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=option)
wait = WebDriverWait(wd, 40)
wd.get('https://www.bedbathandbeyond.com/store/category/kitchen/trash-recycling/14367')
wait.until(EC.element_to_be_clickable((By.XPATH, "/body/div[3]/div[8]/div[1]/div[3]/div[2]/amp-list/div/div/button[2]")))
wd.find_element(By.XPATH,'/body/div[3]/div[8]/div[1]/div[3]/div[2]/amp-list/div/div/button[2]').click()
# class="plpPage plpNext flex mid pwaOnly "
time.sleep(15)
wd.quit()
EDIT:
I think it is some shadow root. I dont know much but tried this code but it did not worked
def expand_shadow_element(element):
shadow_root = wd.execute_script('return arguments[0].shadowRoot', element)
return shadow_root
wd.get('https://www.bedbathandbeyond.com/store/category/kitchen/trash-recycling/14367')
wait.until(EC.element_to_be_clickable((By.XPATH, '/body/div[3]/div[8]/div[1]/div[3]/div[2]/amp-list/div/div')))
ele = wd.find_element(By.XPATH, '/body/div[3]/div[8]/div[1]/div[3]/div[2]/amp-list/div/div')
root = expand_shadow_element(ele)
It gives Timeout error on the wait

you could use javascript instead..this will click the next button
driver.execute_script(
'document.querySelector("#wmHostPrimary").shadowRoot.querySelector("button.plpPage.plpNext.flex.mid.pwaOnly").click()')

How to retrieve data from the popup after clickable element trigger with WebDriverWait in Selenium Python?

I need to scrape the image src from this popup. I have coded that but getting "AttributeError: 'NoneType' object has no attribute 'findElements'.
Here is the code.
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
from chromedriver_py import binary_path
import time
from time import sleep
url = 'https://www.macys.com/shop/product/black-tape-printed-contrast-trim-cardigan?ID=11398979&CategoryID=260&isDlp=true&swatchColor=Neutral%20Animal'
options = Options()
options.add_argument('--disable-blink-features=AutomationControlled')
driver = webdriver.Chrome(options=options, executable_path=binary_path)
wait = WebDriverWait(driver, 10)
driver.get(url)
sizechart_popup = wait.until(EC.element_to_be_clickable((By.XPATH, './/*[#class="sc-link"]'))).click()
sizechart = sizechart_popup.findElements(By.XPATH('.//*[#id="sizeImg"]/img')).get_attribute("src");
print(sizechart)
# Sleep of 10 seconds irrespective of whether element is present or not
time.sleep(50)
# Free up the resources
driver.close()
Thanks in advance

Try using all the available element identifiers if one doesn't work, it worked with css_selector.
sizechart = driver.find_element_by_css_selector('#sizeImg > img').get_attribute("src")
print(sizechart)
#Output:
https://assets.macysassets.com/dyn_img/size_charts/4011861.gif

Still no changes in google colab : selenium in python

import time
import sys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.options import Options
options = webdriver.ChromeOptions()
options.add_argument('--headless')
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
driver=webdriver.Chrome('chromedriver',options=options)
driver.get("https://rentry.co/wftw8/edit")
driver.implicitly_wait(20)
#print (driver.page_source)
try:
# here I selected the **span** element that I talk above
span = driver.find_element_by_xpath('//*[#id="text"]/div/div[5]/div[1]/div/div/div/div[5]/pre/span')
# change the innerText thwough js
driver.execute_script('arguments[0].innerText="Hello boy"', span)
# just wait for the id_edit_code to be present
edit = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.ID, "id_edit_code")))
edit.send_keys("iRfiNq6M")
# and same as you had it
#driver.find_element_by_id("submitButton").send_keys(Keys.ENTER)
#driver.find_element_by_link_text("Save").click()
driver.find_element_by_id("submitButton").click()
except:
print("Oops!", sys.exc_info()[0], "occurred.")
finally:
driver.close()
print("done")
There is no exception but still updating of the text is not reflecting in the url?
Even though there is timer which is enough for the whole code to get processed.
Then also there is no updation.
Currently the text is Hello if u visit the URL but I want it to be Hello boy using selenium , which is done by the below code line:
span = driver.find_element_by_xpath('//*[#id="text"]/div/div[5]/div[1]/div/div/div/div[5]/pre/span')
# change the innerText thwough js
driver.execute_script('arguments[0].innerText="Hello boy"', span)
But no updation!!?