We Keep Coding

Issues scraping a dynamic table displaying available and booked time slots

I want to scrape the following website:https://padelbox.de/koeln-weiden/padelplatz-buchen. I want to scrape the planning tool every day and see what slots are booked and which are not. However, using the code below an error code suggesting the values are not found. Can anyone help me with this?
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as ec
import requests
import time
website = 'https://padelbox.de/koeln-weiden/padelplatz-buchen'
path = '/Users/joeplamers/Downloads/chromedriver_mac_arm64/chromedriver'
# Initialize the webdriver
driver = webdriver.Chrome(path)
# Open the website
driver.get(website)
#close the cookie pop-up and maximize window
all_matches_button = driver.find_element(By.XPATH, '//a[#class="_brlbs-btn _brlbs-btn-accept- all _brlbs-cursor"]')
all_matches_button.click()
driver.maximize_window()
wait = WebDriverWait(driver, 60)
wait.until(ec.presence_of_element_located((By.CSS_SELECTOR,'[data-state="booked"]')))
booked_elements = driver.find_elements(By.CSS_SELECTOR,'[data-state="booked"]')
print(booked_elements)
#Close the browser
driver.quit()

The desired elements are within an <iframe> so you have to:
Induce WebDriverWait for the desired frame to be available and switch to it.
Induce WebDriverWait for the desired element to be visible.
You can use the following locator strategies:
driver.get("https://padelbox.de/koeln-weiden/padelplatz-buchen")
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "a._brlbs-btn-accept-all"))).click()
WebDriverWait(driver, 20).until(EC.frame_to_be_available_and_switch_to_it((By.CSS_SELECTOR,"iframe[src^='https://www.eversports.de/widget']")))
print(len(WebDriverWait(driver, 20).until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, "td[data-state='booked'][data-date]")))))
Note : You have to add the following imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
Console Output:
318
Update
Here's the complete code:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('excludeSwitches', ['enable-logging'])
options.add_experimental_option('useAutomationExtension', False)
options.add_argument('--disable-blink-features=AutomationControlled')
s = Service('C:\\BrowserDrivers\\chromedriver.exe')
driver = webdriver.Chrome(service=s, options=options)
driver.get("https://padelbox.de/koeln-weiden/padelplatz-buchen")
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "a._brlbs-btn-accept-all"))).click()
WebDriverWait(driver, 20).until(EC.frame_to_be_available_and_switch_to_it((By.CSS_SELECTOR,"iframe[src^='https://www.eversports.de/widget']")))
print(len(WebDriverWait(driver, 20).until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, "td[data-state='booked'][data-date]")))))
driver.quit()

website search bar doesn't work for python selenium

I would like auto-click the website and search for the information, but somehow the website cannot search, and keep loading. Or just close quickly after it print the key in search bar.
I would like auto-click the website and search for the information, and I tried:
import selenium
import pandas as pd
import numpy as np
import platform
import time
import random
from os import getcwd
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
options = webdriver.ChromeOptions()
options.add_argument("--no-sandbox")
options.add_argument("--disable-dev-shm-usage")
options.add_argument("--disable-notification")
options.add_argument("--disable-infobars")
options.add_experimental_option('excludeSwitches', ['enable-logging'])
options.add_argument("--remote-debugging-port=9230")
#options.add_argument("--headless")
url = 'https://vip.stock.finance.sina.com.cn/mkt/#hs_z'
driver.get(url)
w = WebDriverWait(driver, 10)
w.until(EC.presence_of_element_located((By.XPATH, '//*[#id="inputSuggest"]')))
driver.find_element('xpath', '//*[#id="inputSuggest"]').clear()
driver.find_element('xpath', '//*[#id="inputSuggest"]').send_keys('sz111973'))
driver.find_element('xpath', '//*[#id="SSForm"]/input[3]').click()
But somehow the website cannot search, and keep loading. Or just close quickly after it print the key in search bar.
Any help will be appreciated! Thanks.

There are several issues here:
to prevent site from very long loading you can use eager pageLoadStrategy.
I see redundant ) at the end of this line driver.find_element('xpath', '//*[#id="inputSuggest"]').send_keys('sz111973'))
The following code works perfect:
from selenium import webdriver
from selenium.webdriver import DesiredCapabilities
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
caps = DesiredCapabilities().CHROME
caps["pageLoadStrategy"] = "eager"
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(service=webdriver_service, options=options, desired_capabilities=caps,)
url = 'https://vip.stock.finance.sina.com.cn/mkt/#hs_z'
driver.get(url)
wait = WebDriverWait(driver, 20)
input = wait.until(EC.element_to_be_clickable((By.ID, 'inputSuggest')))
input.clear()
input.send_keys('sz111973')
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, '#inputSuggest +input'))).click()

Why doesn’t Selenium find an element with this XPath expression?

I’m trying to find an element with this XPath expression:
/html/body/div/div[1]/div/div/div[2]/div/div/div/div[2]/form/div[1]/div[3]/div/input
But Selenium can’t find one.
The page I’m trying to access - https://account.aax.com/en-US/login/
I already tried to follow this path by myself, and it’s fine.

You are missing a delay. WebDriverWait expected_conditions should be used for that.
You have to improve your locators.
The following code works:
from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(options=options, service=webdriver_service)
wait = WebDriverWait(driver, 20)
actions = ActionChains(driver)
url = "https://account.aax.com/en-US/login/"
driver.get(url)
wait.until(EC.element_to_be_clickable((By.XPATH, "//input[#class='MuiInputBase-input MuiOutlinedInput-input']"))).click()

Selenium NoSuchElementException: Message: no such element

I am struggling with Selenium
for the url: https://pubchem.ncbi.nlm.nih.gov/compound/2078
I am trying to click the button Download, but it doesn't find the element.
My code:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.chrome.options import Options
from ipykernel import kernelapp as app
import time
options = webdriver.ChromeOptions()
driver_path = 'C:\\Users\\Idener\\Downloads\\chromedriver_win32\\chromedriver.exe'
driver = webdriver.Chrome(driver_path, options=options)
url = f"https://pubchem.ncbi.nlm.nih.gov/compound/2078"
driver.get(url)
driver.find_element_by_xpath("//*[#id='"'page-download-btn'"']").click()
enter image description here

Your XPath is not valid. You don't need so much quotes
driver.find_element_by_xpath("//*[#id='page-download-btn']").click()

You are missing a delay.
Element should clicked only when it is completely rendered and ready to accept a click event. WebDriverWait expected_conditions explicit waits should be used for that.
Also, no need to add f before URL value and '"' instead of ' in XPath expression.
The following code will work for you:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.chrome.options import Options
from ipykernel import kernelapp as app
import time
options = webdriver.ChromeOptions()
driver_path = 'C:\\Users\\Idener\\Downloads\\chromedriver_win32\\chromedriver.exe'
driver = webdriver.Chrome(driver_path, options=options)
url = "https://pubchem.ncbi.nlm.nih.gov/compound/2078"
driver.get(url)
wait = WebDriverWait(driver, 10)
wait.until(EC.element_to_be_clickable((By.ID, "page-download-btn"))).click()

Why isn't selenium successfully clicking my download link?

I am trying to use selenium to download an excel file from a website. I am not sure why the code isn't allowing me to download it. I get an exit code 0 so everything ran successfully but I am not seeing the file in my downloads.
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
def scrape_mclellan_website():
url = 'https://www.mcoscillator.com/market_breadth_data/'
s = Service(ChromeDriverManager().install())
op = webdriver.ChromeOptions()
op.add_argument('headless')
driver = webdriver.Chrome(service=s)
driver.get(url)
download_link = driver.find_element(by=By.XPATH, value='//*[#id="data_table"]/a[1]/img')
download_link.click()
scrape_mclellan_website()

How to fix?
Wait until the element you try to click is present and click the <a> not the <img>:
download_link = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, '//*[#id="data_table"]/a[1]')))
download_link.click()
Set your preferences for donwload folder and take care window is opening in right size:
prefs = {'download.default_directory':'ENTER PATH TO DOWNLOAD FOLDER'}
options = webdriver.ChromeOptions()
options.add_argument("--window-size=1920,1080")
options.add_argument("--start-maximized")
options.add_argument("--headless")
options.add_experimental_option("prefs",prefs)
Example (selenium 4)
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
service = Service(executable_path='ENTER YOUR PATH TO CHROMEDRIVER')
prefs = {'download.default_directory':'ENTER PATH TO DOWNLOAD FOLDER'}
options = webdriver.ChromeOptions()
options.add_argument("--window-size=1920,1080")
options.add_argument("--start-maximized")
options.add_argument("--headless")
options.add_experimental_option("prefs",prefs)
driver = webdriver.Chrome(service=service, options=options)
driver.get('https://www.mcoscillator.com/market_breadth_data/')
download_link = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, '//*[#id="data_table"]/a[1]')))
download_link.click()