We Keep Coding

Are there any methods in Selenium for waiting until a page is fully loaded?

I am trying to find a method that waits when all elements on the page will be fully loaded.
I found out a good idea and it is working very well:
def waiting_new_page(link: WebElement) -> None:
waiting_update = True
while waiting_update:
try:
link.find_element(By.ID, "does not matter")
except NoSuchElementException:
sleep(1)
except StaleElementReferenceException:
waiting_update = False
options = webdriver.ChromeOptions()
options.add_argument("window-size=800,600")
browser = webdriver.Chrome(options=options)
link = browser.find_element(
By.NAME,
"thePage:SiteTemplate:theForm:Continue"
)
link.click()
waiting_new_page(link)
Are there any ready methods in Selenium to wait until page is fully loaded?

You can use WebDriverWait function in Selenium
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.common.by import By
WebDriverWait(browser, timeout=100).until(EC.presence_of_element_located((By.CSS_SELECTOR, 'body')))
Or instead of body you can put your selector you want wait until to load.

Can I use selenium to set the dates in the Microsoft Power Bi dashboard?

I try to scrape data from the public Microsoft Power Bi Dashboard (4th page).
But unfortunately, I can't understand how with selenium I can change periods of time.
Tell me, please, is this even possible using python + selenium? Maybe move these sliders, or enter dates into inputs.
Thanks.
Dashboard screen
Here is my code to load the dashboard page:
from selenium import webdriver
import time
fp = webdriver.FirefoxProfile()
url='https://app.powerbi.com/view?r=eyJrIjoiNjIwNzg5NzQtNzRlYS00YzFmLWJiNTUtOTM2MGEwY2FjOGJlIiwidCI6ImE3NWRkYWZlLWQ2MmYtNGIxOS04NThhLTllYzFhYjI1NDdkNCIsImMiOjl9'
driver = webdriver.Firefox(firefox_profile=fp)
driver.get(url)
time.sleep(5)
driver.find_element_by_xpath('//*[#id="embedWrapperID"]/div[2]/logo-bar/div/div/div/logo-bar-navigation/span/button[2]/i').click()
driver.find_element_by_xpath('//*[#id="embedWrapperID"]/div[2]/logo-bar/div/div/div/logo-bar-navigation/span/button[2]/i').click()
driver.find_element_by_xpath('//*[#id="embedWrapperID"]/div[2]/logo-bar/div/div/div/logo-bar-navigation/span/button[2]/i').click()

I was able to get it to set the date by clearing the existing date, and then using a date string + tab to refresh the graph, is that what you are looking for?
I also packaged up my attempts to click on the various items with a wait function so I'm not waiting some unnecessary amount of time before something shows up on the screen.
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
import time
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common import exceptions
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.keys import Keys
def InputByXPATH(NameOfObject, WhatToSend):
try:
item = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, NameOfObject)))
item.click()
item.send_keys(Keys.CONTROL, 'a')
item.send_keys(WhatToSend)
except TimeoutException as e:
print("InputByXPATH Error: Couldn't input by XPATH on: " + str(NameOfObject))
pass
def ClickByXPATH(NameOfObject):
try:
item = WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.XPATH, NameOfObject)))
item.click()
except TimeoutException as e:
print("ClickByXpath Error: Couldn't Click by XPATH on: " + str(NameOfObject))
pass
driver = webdriver.Chrome()
driver.maximize_window()
url='https://app.powerbi.com/view?r=eyJrIjoiNjIwNzg5NzQtNzRlYS00YzFmLWJiNTUtOTM2MGEwY2FjOGJlIiwidCI6ImE3NWRkYWZlLWQ2MmYtNGIxOS04NThhLTllYzFhYjI1NDdkNCIsImMiOjl9'
driver.get(url)
ClickByXPATH('//*[#id="embedWrapperID"]/div[2]/logo-bar/div/div/div/logo-bar-navigation/span/button[2]/i')
ClickByXPATH('//*[#id="embedWrapperID"]/div[2]/logo-bar/div/div/div/logo-bar-navigation/span/button[2]/i')
InputByXPATH('/html/body/div[1]/report-embed/div/div/div[1]/div/div/div/exploration-container/div/div/div/exploration-host/div/div/exploration/div/explore-canvas/div/div[2]/div/div[2]/div[2]/visual-container-repeat/visual-container[8]/transform/div/div[3]/div/visual-modern/div/div/div[2]/div/div[1]/div/div[1]/input', '2/1/2022' + Keys.TAB )
time.sleep(10)

click() function in selenium not interactable. Python [duplicate]

I understand this question has been asked but I need some solution for this error:
Traceback (most recent call last):
File "goeventz_automation.py", line 405, in <module>
if login(driver) is not None:
File "goeventz_automation.py", line 149, in login
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//a[#track-element='header-login']"))).click()
File "/usr/local/lib/python3.6/dist-packages/selenium/webdriver/support/wait.py", line 80, in until
raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message:
This is the code where its getting error:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import TimeoutException
import urllib.request as request
import urllib.error as error
from PIL import Image
from selenium.webdriver.chrome.options import Options
import datetime as dt
import time
from common_file import *
from login_credentials import *
def login(driver):
global _email, _password
if waiter(driver, "//a[#track-element='header-login']") is not None:
#login = driver.find_element_by_xpath("//a[#track-element='header-login']")
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//a[#track-element='header-login']"))).click()
#login.click()
if waiter(driver,"//input[#id='user_email']") is not None:
email = driver.find_element_by_xpath("//input[#id='user_email']")
password = driver.find_element_by_xpath("//input[#id='password']")
email.send_keys(_email)
password.send_keys(_password)
driver.find_element_by_xpath("//button[#track-element='click-for-login']").click()
return driver
else:
print("There was an error in selecting the email input field. It may be the page has not loaded properly.")
return None
else:
print("There was an error in selecting the header-login attribute on the page.")
return None
if __name__ == '__main__':
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
driver = webdriver.Chrome('/usr/bin/chromium/chromedriver',chrome_options=chrome_options)
#d.get('https://www.google.nl/')
#driver = webdriver.Chrome()
driver.maximize_window()
driver.get('https://www.goeventz.com/')
if login(driver) is not None:
print(create_event(driver))
I think there is some problem with Keys.ENTER, but I don't know how to solve this. I have tried every possible solution.............

This error message...
selenium.common.exceptions.ElementNotInteractableException: Message: element not interactable
...implies that the desired element was not interactable when you tried to invoke click() on it.
A couple of facts:
When you initialize the Chrome browser always in maximized mode.
You can disable-extensions.
You need to disable-infobars as well.
I have used the same xpath which you have constructed and you can use the following solution:
Code Block:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
options = webdriver.ChromeOptions()
options.add_argument("start-maximized");
options.add_argument("disable-infobars")
options.add_argument("--disable-extensions")
driver = webdriver.Chrome(chrome_options=options, executable_path=r'C:\Utility\BrowserDrivers\chromedriver.exe')
driver.get("https://www.goeventz.com/")
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, "//a[#track-element='header-login']"))).click()
Browser Snapshot:

copy full xpath instead of copying only xpath. It will work

Instead of using login.send_keys(Keys.ENTER) you should use selenium click() method which would work fine for you.
You can check first if the element is clickable first and then you can click on it.
Like:
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//a[#track-element='header-login']"))).click()

Overview
It seems like you're having an XPATH problem finding the "Submit" button or your Submit button is not clickable, or your Submit button has some client side events attached to it (javascript/etc) that are required in order to effectively submit the page.
Calling the pw.submit() method in most cases should get rid of the need to wait for the submit button to become clickable and avoid any issues in locating the button in most cases. On many other websites, some of the necessary back-end processes are primed by client-side activities that are performed after the "submit" button is actually clicked (although on a side-note this is not considered best-practice because it makes the site less accessible, etc, I digress). Above all, it's important to watch your script execute and make sure that you're not getting any noticeable errors displayed on the webpage about the credentials that you're submitting.
Also, however, some websites require that you add a certain minimum amount of time between the entry of the username, password, and submitting the page in order for it to be considered a valid submitting process. I've even run in to websites that require you to use send_keys 1 at a time for usernames and passwords to avoid some anti-scraping technologies they employ. In these cases, I usually use the following between the calls:
from random import random, randint
def sleepyTime(first=5, second=10):
# returns the value of the time slept (as float)
# sleeps a random amount of time between the number variable in first
# and the number variable second (in seconds)
sleepy_time = round(random() * randint(first, second), 2)
sleepy_time = sleepy_time if sleepy_time > first else (first + random())
sleep(sleepy_time)
return sleepy_time
I don't see what use you have for making the _email and _password variables global, unless they are being changed somewhere in the login function and you want that change to be precipitated out to the other scopes.
How I would try to solve it
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import NoSuchElementException, TimeoutException
TIME_TIMEOUT = 20 # Twenty-second timeout default
def eprint(*args, **kwargs):
""" Prints an error message to the user in the console (prints to sys.stderr), passes
all provided args and kwargs along to the function as usual. Be aware that the 'file' argument
to print can be overridden if supplied again in kwargs.
"""
print(*args, file=sys.stderr, **kwargs)
def login(driver):
global _email, _password
try:
email = WebDriverWait(driver, TIME_TIMEOUT).until(EC.presence_of_element_located((By.XPATH, "//input[#id='user_email']")))
pw = WebDriverWait(driver, TIME_TIMEOUT).until(EC.presence_of_element_located((By.XPATH, "//input[#id='password']"))
pw.submit()
# if this doesn't work try the following:
# btn_submit = WebDriverWait(driver, TIME_TIMEOUT).until(EC.element_to_be_clickable((By.XPATH, "//button[#track-element='click-for-login']"))
# btn_submit.click()
# if that doesn't work, try to add some random wait times using the
# sleepyTime() example from above to add some artificial waiting to your email entry, your password entry, and the attempt to submit the form.
except NoSuchElementException as ex:
eprint(ex.msg())
except TimeoutException as toex:
eprint(toex.msg)
if __name__ == '__main__':
driver = webdriver.Chrome('/usr/bin/chromium/chromedriver',chrome_options=chrome_options)
#d.get('https://www.google.nl/')
#driver = webdriver.Chrome()
driver.maximize_window()
driver.get('https://www.goeventz.com/')
if login(driver) is not None:
print(create_event(driver))

For headless chrome browser you need to provide window size as well in chrome options.For headless browser selenium unable to know what your window size.Try that and let me know.
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.add_argument('window-size=1920x1480')

I faced this error as well. Now check your browser if the element is inside the iframe. If so, use driver.find_element(By.CSS_SELECTOR, "#payment > div > div > iframe") and driver.switch_to.frame(iframe) Then you will be able to work out.

How to make Selenium only click a button and nothing else? Inconsistent clicking

My goal: to scrape the amount of projects done by a user on khan academy.
To do so I need to parse the profile user page. But I need to click on show more to see all the project a user had done and then scrape them.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException,StaleElementReferenceException
from bs4 import BeautifulSoup
# here is one example of a user
driver = webdriver.Chrome()
driver.get('https://www.khanacademy.org/profile/trekcelt/projects')
# to infinite click on show more button until there is none
while True:
try:
showmore_project=WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME,'showMore_17tx5ln')))
showmore_project.click()
except TimeoutException:
break
except StaleElementReferenceException:
break
# parsing the profile
soup=BeautifulSoup(driver.page_source,'html.parser')
# get a list of all the projects
project=soup.find_all(class_='title_1usue9n')
# get the number of projects
print(len(project))
This code return 0 for print(len(project)). And that's not normal because when you manually check https://www.khanacademy.org/profile/trekcelt/projects you can see there that the amount of projects is definetly not 0.
The weird thing: at first, you can see (with the webdriver) that this code is working fine and then selenium clicks on something else than the show more button, it click on one of the project's link for example and thus change the page and that's why we get 0.
I don't understand how to correct my code so selenium is only clicking on the right button and nothing else.

Check out the following implementation to get the desired behavior. When the script is running, take a closer look at the scroll bar to see the progress.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup
with webdriver.Chrome() as driver:
wait = WebDriverWait(driver,10)
driver.get('https://www.khanacademy.org/profile/trekcelt/projects')
while True:
try:
showmore = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,'[class^="showMore"] > a')))
driver.execute_script("arguments[0].click();",showmore)
except Exception:
break
soup = BeautifulSoup(driver.page_source,'html.parser')
project = soup.find_all(class_='title_1usue9n')
print(len(project))
Another way would be:
while True:
try:
showmore = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,'[class^="showMore"] > a')))
showmore.location_once_scrolled_into_view
showmore.click()
wait.until(EC.invisibility_of_element_located((By.CSS_SELECTOR,'[class^="spinnerContainer"] > img[class^="loadingSpinner"]')))
except Exception:
break
Output at this moment:
381

I have modified the accepted answer to improve the performance of your script. Comment on how you can achieve it is in the code
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException, StaleElementReferenceException
from bs4 import BeautifulSoup
import time
start_time = time.time()
# here is one example of a user
with webdriver.Chrome() as driver:
driver.get('https://www.khanacademy.org/profile/trekcelt/projects')
# This code will wait until the first Show More is displayed (After page loaded)
showmore_project = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME,
'showMore_17tx5ln')))
showmore_project.click()
# to infinite click on show more button until there is none
while True:
try:
# We will retrieve and click until we do not find the element
# NoSuchElementException will be raised when we reach the button. This will save the wait time of 10 sec
showmore_project= driver.find_element_by_css_selector('.showMore_17tx5ln [role="button"]')
# Using a JS to send the click will avoid Selenium to through an exception where the click would not be
# performed on the right element.
driver.execute_script("arguments[0].click();", showmore_project)
except StaleElementReferenceException:
continue
except NoSuchElementException:
break
# parsing the profile
soup=BeautifulSoup(driver.page_source,'html.parser')
# get a list of all the projects
project=soup.find_all(class_='title_1usue9n')
# get the number of projects
print(len(project))
print(time.time() - start_time)
Execution Time1: 14.343502759933472
Execution Time2: 13.955228090286255
Hope this help you!

Selenium Webdriver - Python - leboncoin - pb to select a button with an accent

I am trying to fill in a form automatically on the following website: 'https://www.leboncoin.fr/'
I have recorded a script with Selenium IDE.
I have a function to connect automatically by clicking on the button 'Se connecter' and filling my pwd and username. It works fine
I have set up specific credential for this topic
email: thecoingood#gmail.com
pwd: thecoingood1
the code is
# -*- coding: utf-8 -*-
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import NoAlertPresentException
import unittest, time, re
class Connectionwebdriver2(unittest.TestCase):
def setUp(self):
self.driver = webdriver.Safari()
self.driver.implicitly_wait(30)
self.driver.maximize_window()
self.base_urldr = "https://compteperso.leboncoin.fr/account/index.html"
self.verificationErrors = []
self.accept_next_alert = True
def test_connectionwebdriver2(self):
driver = self.driver
driver.get(self.base_urldr)
driver.find_element_by_name("st_username").clear()
driver.find_element_by_name("st_username").send_keys("thecoingood#gmail.com ")
driver.find_element_by_name("st_passwd").clear()
driver.find_element_by_name("st_passwd").send_keys("thecoingood1")
driver.find_element_by_id("connect_button").click()
#driver.get("https://www2.leboncoin.fr/ai?ca=15_s")
my_annonce = WebDriverWait(self.driver, 10)\
.until(EC.element_to_be_clickable((By.LINK_TEXT, "Supprimer")))
my_annonce.click()
#time.sleep(10)
#driver.find_element_by_link_text("Supprimer").click()
#WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//a[#href='//https://compteperso.leboncoin.fr/account/index.html?ca=12_s' and contains(.,'posez une annonce')]"))).click()
#Select(driver.find_element_by_id("category")).select_by_visible_text('Locations')
#Select(driver.find_element_by_id('cat10')).select()
def is_element_present(self, how, what):
try: self.driver.find_element(by=how, value=what)
except NoSuchElementException as e: return False
return True
def is_alert_present(self):
try: self.driver.switch_to_alert()
except NoAlertPresentException as e: return False
return True
def close_alert_and_get_its_text(self):
try:
alert = self.driver.switch_to_alert()
alert_text = alert.text
if self.accept_next_alert:
alert.accept()
else:
alert.dismiss()
return alert_text
finally: self.accept_next_alert = True
def tearDown(self):
self.driver.quit()
self.assertEqual([], self.verificationErrors)
if __name__ == "__main__":
unittest.main(verbosity=2)
Once connected, I am redirected to
https://compteperso.leboncoin.fr/account/index.html?ca=12_s
(question: is the object used in selenium updated with this new address or still sticks with the initial one which may create the issue)
when I try to click on the
Déposez une annonce
with this code
driver.find_element_by_link_text(u"Déposez une annonce").click()
Nothing happens (no error).
I believe this is related to the fact the link is not yet visible.
I have tried to add time.sleep() and also read
How can I get Selenium Web Driver to wait for an element to be accessible, not just present?
but coud not resolve this. I could add a direct link to the page, but I would like to understand.
Thanks in advance

As per your question to click on the tab / link with text as Déposez une annonce you can use the following line of code :
To click the TAB with text as Déposez une annonce use :
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//li[#class='deposer']/a[contains(#href,'//www2.leboncoin.fr/ai?ca=') and contains(.,'une annonce')]"))).click()
To click the BUTTON with text as Déposez une annonce use :
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//div[#class='create']/a[contains(#href,'//www2.leboncoin.fr/ai?ca=') and contains(.,'une annonce')]"))).click()