Chrome and Edge Selenium crashes after the for-loop - python

I am writing an automation script that would facilitate job application process. What I want to do is to open several new tabs within a browser and then successively apply for them. All was going well until the browser shuts down after successfully completing the iteration without raising any exceptions. I want to keep the tabs open so I can apply for the job ads and make a record of it.
I have already tried adding additional time space to accommodate for page load-up; cookies deleteion is out of quesiton because I have to be logged in. Additionally, I have reduced the number of links. Tested on MS Edge and Chrome.
The "get_links" function delivers corresponding links to job ads.
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait, Select
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
browser = webdriver.Chrome("chromedriver.exe")
delay = 5
# Job Preferences
preferences = {
"position-title" : "financial analyst"
}
# URLs
url_home = "https://www.reed.co.uk"
# url_example = """https://www.reed.co.uk/jobs/financial-analyst-jobs?pageno=1"""
# Logging In
browser.get(url_home)
try:
element_present = EC.presence_of_element_located((By.XPATH, """//*[#id="onetrust-accept-btn-handler"]"""))
WebDriverWait(browser, delay).until(element_present)
except TimeoutException:
print("the page has not loaded up.")
cookies = browser.find_element_by_xpath("""//*[#id="onetrust-accept-btn-handler"]""")
cookies.click()
signIn_button = browser.find_element_by_xpath("""//*[#id="site-header"]/div[2]/div[1]/ul/li[2]/a""")
signIn_button.click()
try:
element_present = EC.presence_of_element_located((By.XPATH, """//*[#id="signin_email"]"""))
WebDriverWait(browser, delay).until(element_present)
except TimeoutException:
print("the page has not loaded up.")
username_ = browser.find_element_by_xpath("""//*[#id="signin_email"]""")
username_.send_keys("email")
password_ = browser.find_element_by_xpath("""//*[#id="signin_password"]""")
password_.send_keys("password")
signIn_ = browser.find_element_by_xpath("""//*[#id="signin_button"]""")
signIn_.click()
# Query search after having logged in.
try:
element_present = EC.presence_of_element_located((By.XPATH, """//*[#id="keywords"]"""))
WebDriverWait(browser, delay).until(element_present)
except TimeoutException:
print("the page has not loaded up.")
search_query = browser.find_element_by_xpath("""//*[#id="keywords"]""")
search_query.send_keys(preferences.get("position-title"))
search_button = browser.find_element_by_xpath("""//*[#id="main-search"]/div[1]/div[3]/button""")
search_button.click()
# Filters and job type selection
try:
element_present = EC.presence_of_element_located((By.XPATH, """//*[#id="refine-salary-from"]"""))
WebDriverWait(browser, delay).until(element_present)
except TimeoutException:
print("the page has not loaded up.")
salaryRange = Select(browser.find_element_by_xpath("""//*[#id="refine-salary-from"]"""))
salaryRange.select_by_value('28000')
browser.implicitly_wait(2)
parent_tab = browser.current_window_handle
for link_ in get_links.get_joblinks(url_home, browser.current_url).Link[0:5]:
sleep(2)
browser.execute_script('''window.open("{}");'''.format(link_))
sleep(2)

Related

Unable to locate the element on an angular website

I have tried the below code and it is always timing out. But when I look it up with the browser inspector, I can see the Username input element.
I have also tried to find it by ID but not able to.
Tried almost all of the existing questions/solutions on this forum but was unable to figure it out.
driver.get("https://www.dat.com/login")
time.sleep(5)
driver.find_element_by_css_selector("a[href*='https://power.dat.com/']").click()
time.sleep(5)
# explicitly waiting until input element load
try:
WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.NAME, "username"))
)
except TimeoutException:
print(
"Waited 10 seconds for the username input to load but did not happen..."
)
except Exception as e:
print(f"Exception while waiting for username input to appear: \n {e}")
sys.exit(1)
2 issues here:
After clicking on "DAT Power" on the first page a new tab is opened. To continue working there you need to switch the driver to the second tab.
You will probably want to enter a text into the username there. If so you need to wait for element clickability, not presence only.
Also, the current Selenium version (4.x) no more supports find_element_by_* methods, the new style should be used, as following presented.
The following code works:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(options=options, service=webdriver_service)
wait = WebDriverWait(driver, 10)
url = "https://www.dat.com/login"
driver.get(url)
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "a[href*='https://power.dat.com/']"))).click()
new_tab = driver.window_handles[1]
driver.switch_to.window(new_tab)
wait.until(EC.element_to_be_clickable((By.NAME, "username"))).send_keys("name#gmail.com")
The result is

In selenium how to avoid TimeoutException error after multiple hits in sequence?

With Python3 and selenium I want to automate the search on a public information site. In this site it is necessary to enter the name of a person, then select the spelling chosen for that name (without or with accents or name variations), access a page with the list of lawsuits found and in this list you can access the page of each case.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException, NoSuchElementException
from selenium.webdriver.common.keys import Keys
import time
import re
Name that will be searched
name = 'JOSE ROBERTO ARRUDA'
Create path, search start link, and empty list to store information
firefoxPath="/home/abraji/Documentos/Code/geckodriver"
link = 'https://ww2.stj.jus.br/processo/pesquisa/?aplicacao=processos.ea'
processos = []
Call driver and go to first search page
driver = webdriver.Firefox(executable_path=firefoxPath)
driver.get(link)
Position cursor, fill and click
WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.CSS_SELECTOR, '#idParteNome'))).click()
time.sleep(1)
driver.find_element_by_xpath('//*[#id="idParteNome"]').send_keys(name)
time.sleep(6)
WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.CSS_SELECTOR, '#idBotaoPesquisarFormularioExtendido'))).click()
Mark all spelling possibilities for searching
WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.CSS_SELECTOR, '#idBotaoMarcarTodos'))).click()
WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.CSS_SELECTOR, '#idBotaoPesquisarMarcados'))).click()
time.sleep(1)
Check how many pages of data there are - to be used in "for range"
capta = driver.find_element_by_xpath('//*[#id="idDivBlocoPaginacaoTopo"]/div/span/span[2]').text
print(capta)
paginas = int(re.search(r'\d+', capta).group(0))
paginas = int(paginas) + 1
print(paginas)
Capture routine
for acumula in range(1, paginas):
# Fill the field with the page number and press enter
driver.find_element_by_xpath('//*[#id="idDivBlocoPaginacaoTopo"]/div/span/span[2]/input').send_keys(acumula)
driver.find_element_by_xpath('//*[#id="idDivBlocoPaginacaoTopo"]/div/span/span[2]/input').send_keys(Keys.RETURN)
time.sleep(2)
# Captures the number of processes found on the current page - qt
qt = driver.find_element_by_xpath('//*[#id="idDivBlocoMensagem"]/div/b').text
qt = int(qt) + 2
print(qt)
# Iterate from found number of processes
for item in range(2, qt):
# Find the XPATH of each process link - start at number 2
vez = '//*[#id="idBlocoInternoLinhasProcesso"]/div[' + str(item) + ']/span[1]/span[1]/span[1]/span[2]/a'
# Access the direct link and click
time.sleep(2)
WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.XPATH, vez))).click()
time.sleep(1)
# Run tests to get data
try:
num_unico = driver.find_element_by_xpath('//*[#id="idProcessoDetalhesBloco1"]/div[6]/span[2]/a').text
except NoSuchElementException:
num_unico = "sem_numero_unico"
try:
nome_proc = driver.find_element_by_xpath('//*[#id="idSpanClasseDescricao"]').text
except NoSuchElementException:
nome_proc = "sem_nome_encontrado"
try:
data_autu = driver.find_element_by_xpath('//*[#id="idProcessoDetalhesBloco1"]/div[5]/span[2]').text
except NoSuchElementException:
data_autu = "sem_data_encontrada"
# Fills dictionary and list
dicionario = {"num_unico": num_unico,
"nome_proc": nome_proc,
"data_autu": data_autu
}
processos.append(dicionario)
# Return a page to click on next process
driver.execute_script("window.history.go(-1)")
# Close driver
driver.quit()
After about 30 hits to the direct links with the information of each process I have this error:
---------------------------------------------------------------------------
TimeoutException Traceback (most recent call last)
<ipython-input-10-a901a514bd82> in <module>
16
17 time.sleep(2)
---> 18 WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.XPATH, vez))).click()
19 time.sleep(1)
20
~/Documentos/Code/publique_se/lib/python3.6/site-packages/selenium/webdriver/support/wait.py in until(self, method, message)
78 if time.time() > end_time:
79 break
---> 80 raise TimeoutException(message, screen, stacktrace)
81
82 def until_not(self, method, message=''):
TimeoutException: Message:
Apparently the site gets slow and the script shows error because it can't find the information, right?
Please, on sites where selenium traverses multiple pages is there a way to avoid this kind of error?
Can the website itself slow down when it perceives hits in sequence?
You are using threads to wait which is not a good way and causing errors as you are getting
you are using time.sleep(2) instead use explicit waits of selenium
Example:
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
element = WebDriverWait(driver, 20).until(
EC.element_to_be_clickable((By.XPATH, "myXpath")))
element.click();
Note: You need to change the time from 20 to as per your application approx time you observe

Not able to avoid ElementNotVisibleException even though I'm using a try and except method

I want to download millions of excel files from a website using selenium. My current code tries to handle issues with ElementNotVisibleException, but my "try and except" methods seems to fall short.
I have tried to implement a try and except solution, where if the error message appears I have instructed Selenium to wait untill the "button" appears.
import os
import time
import csv
from tqdm import tqdm
import pandas as pd
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import ElementNotVisibleException
from selenium.common.exceptions import ElementClickInterceptedException
working_directory = r"xx"
os.chdir(working_directory)
options = webdriver.ChromeOptions()
prefs = {
"download.default_directory": r"xx",
"download.prompt_for_download": False,
"download.directory_upgrade": True}
options.add_experimental_option("prefs", prefs)
driver = webdriver.Chrome(r"C:xxx\chromedriver.exe", options = options)
driver.get("website")
# Login
driver.find_element_by_class_name("smallLoginBox").click()
driver.implicitly_wait(1)
time.sleep(2)
driver.find_element_by_id('loginFormUC_loginEmailTextBox').send_keys('EMAIL')
driver.find_element_by_id('loginFormUC_loginPasswordTextBox').send_keys('PASWORD')
driver.find_element_by_xpath("//input[#value='Logg inn']").click()
# Get a custom list of firms
bedrifter = []
with open("./listwithIDs.csv") as csvDataFile:
csvReader = csv.reader(csvDataFile)
for row in csvReader:
bedrifter.append(row[0])
# THE LOOP
for ID in tqdm(bedrifter_gjenstår):
driver.get("website" + ID)
source = driver.page_source
if not "Ingen data" in source: # make sure there is an excel file. If not, loop continues to next ID.
# first click on button "download excel"
try:
driver.find_element_by_id("exportExcel").click()
except ElementNotVisibleException:
WebDriverWait(driver, 120).until(
EC.presence_of_element_located((By.ID, "exportExcel")))
driver.find_element_by_id("exportExcel").click()
except ElementClickInterceptedException:
WebDriverWait(driver, 120).until(
EC.presence_of_element_located((By.ID, "exportExcel")))
driver.find_element_by_id("exportExcel").click()
# second click, choosing what format the excel file should be in
try:
driver.find_element_by_id("mainContentPlaceHolder_mainContentPlaceHolder_mainContentPlaceHolder_AccountingNumberTableUc_excelLinkButton").click()
except ElementNotVisibleException:
WebDriverWait(driver, 120).until(
EC.presence_of_element_located((By.ID, "mainContentPlaceHolder_mainContentPlaceHolder_mainContentPlaceHolder_AccountingNumberTableUc_excelLinkButton")))
driver.find_element_by_id("mainContentPlaceHolder_mainContentPlaceHolder_mainContentPlaceHolder_AccountingNumberTableUc_excelLinkButton").click()
# code to switch between windows to remove download window and continue the code
try:
window_export = driver.window_handles[1]
except IndexError:
time.sleep(3)
print("sleep")
window_export = driver.window_handles[1]
try:
window_main = driver.window_handles[0]
except IndexError:
time.sleep(3)
print("sleep")
window_main = driver.window_handles[0]
driver.switch_to.window(window_export)
driver.close()
driver.switch_to.window(window_main)
I expect the code to download all files if there are any, but either ElementNotVisibleException or ElementClickInterceptedException appears.
To make sure the element is not only present and visible but also clickable check for clicability as a wait condition:
element = WebDriverWait(driver, 120).until(EC.element_to_be_clickable((By.ID, "exportExcel"))).click()
More on Python waits can be found here.
Before perform any operation on the element first check it is visible or not
Here is the example code:
wait = WebDriverWait(self.browser, 15)
wait.until(EC.visibility_of_element_located(("element_path")))
driver.find_element_by_xpath("element_path").click()
You probably get these exceptions because they occur inside an except clause and not inside a try. Please look at the traceback to determine which line threw the exception. This should tell you where the problem lies.
Also, if you want to wait for the element to be visible you should use visibility_of_element_located instead of presence_of_element_located

Wait until page is loaded with Selenium WebDriver for Python

I want to scrape all the data of a page implemented by a infinite scroll. The following python code works.
for i in range(100):
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(5)
This means every time I scroll down to the bottom, I need to wait 5 seconds, which is generally enough for the page to finish loading the newly generated contents. But, this may not be time efficient. The page may finish loading the new contents within 5 seconds. How can I detect whether the page finished loading the new contents every time I scroll down? If I can detect this, I can scroll down again to see more contents once I know the page finished loading. This is more time efficient.
The webdriver will wait for a page to load by default via .get() method.
As you may be looking for some specific element as #user227215 said, you should use WebDriverWait to wait for an element located in your page:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
browser = webdriver.Firefox()
browser.get("url")
delay = 3 # seconds
try:
myElem = WebDriverWait(browser, delay).until(EC.presence_of_element_located((By.ID, 'IdOfMyElement')))
print "Page is ready!"
except TimeoutException:
print "Loading took too much time!"
I have used it for checking alerts. You can use any other type methods to find the locator.
EDIT 1:
I should mention that the webdriver will wait for a page to load by default. It does not wait for loading inside frames or for ajax requests. It means when you use .get('url'), your browser will wait until the page is completely loaded and then go to the next command in the code. But when you are posting an ajax request, webdriver does not wait and it's your responsibility to wait an appropriate amount of time for the page or a part of page to load; so there is a module named expected_conditions.
Trying to pass find_element_by_id to the constructor for presence_of_element_located (as shown in the accepted answer) caused NoSuchElementException to be raised. I had to use the syntax in fragles' comment:
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
driver = webdriver.Firefox()
driver.get('url')
timeout = 5
try:
element_present = EC.presence_of_element_located((By.ID, 'element_id'))
WebDriverWait(driver, timeout).until(element_present)
except TimeoutException:
print "Timed out waiting for page to load"
This matches the example in the documentation. Here is a link to the documentation for By.
Find below 3 methods:
readyState
Checking page readyState (not reliable):
def page_has_loaded(self):
self.log.info("Checking if {} page is loaded.".format(self.driver.current_url))
page_state = self.driver.execute_script('return document.readyState;')
return page_state == 'complete'
The wait_for helper function is good, but unfortunately click_through_to_new_page is open to the race condition where we manage to execute the script in the old page, before the browser has started processing the click, and page_has_loaded just returns true straight away.
id
Comparing new page ids with the old one:
def page_has_loaded_id(self):
self.log.info("Checking if {} page is loaded.".format(self.driver.current_url))
try:
new_page = browser.find_element_by_tag_name('html')
return new_page.id != old_page.id
except NoSuchElementException:
return False
It's possible that comparing ids is not as effective as waiting for stale reference exceptions.
staleness_of
Using staleness_of method:
#contextlib.contextmanager
def wait_for_page_load(self, timeout=10):
self.log.debug("Waiting for page to load at {}.".format(self.driver.current_url))
old_page = self.find_element_by_tag_name('html')
yield
WebDriverWait(self, timeout).until(staleness_of(old_page))
For more details, check Harry's blog.
As mentioned in the answer from David Cullen, I've always seen recommendations to use a line like the following one:
element_present = EC.presence_of_element_located((By.ID, 'element_id'))
WebDriverWait(driver, timeout).until(element_present)
It was difficult for me to find somewhere all the possible locators that can be used with the By, so I thought it would be useful to provide the list here.
According to Web Scraping with Python by Ryan Mitchell:
ID
Used in the example; finds elements by their HTML id attribute
CLASS_NAME
Used to find elements by their HTML class attribute. Why is this
function CLASS_NAME not simply CLASS? Using the form object.CLASS
would create problems for Selenium's Java library, where .class is a
reserved method. In order to keep the Selenium syntax consistent
between different languages, CLASS_NAME was used instead.
CSS_SELECTOR
Finds elements by their class, id, or tag name, using the #idName,
.className, tagName convention.
LINK_TEXT
Finds HTML tags by the text they contain. For example, a link that
says "Next" can be selected using (By.LINK_TEXT, "Next").
PARTIAL_LINK_TEXT
Similar to LINK_TEXT, but matches on a partial string.
NAME
Finds HTML tags by their name attribute. This is handy for HTML forms.
TAG_NAME
Finds HTML tags by their tag name.
XPATH
Uses an XPath expression ... to select matching elements.
From selenium/webdriver/support/wait.py
driver = ...
from selenium.webdriver.support.wait import WebDriverWait
element = WebDriverWait(driver, 10).until(
lambda x: x.find_element_by_id("someId"))
On a side note, instead of scrolling down 100 times, you can check if there are no more modifications to the DOM (we are in the case of the bottom of the page being AJAX lazy-loaded)
def scrollDown(driver, value):
driver.execute_script("window.scrollBy(0,"+str(value)+")")
# Scroll down the page
def scrollDownAllTheWay(driver):
old_page = driver.page_source
while True:
logging.debug("Scrolling loop")
for i in range(2):
scrollDown(driver, 500)
time.sleep(2)
new_page = driver.page_source
if new_page != old_page:
old_page = new_page
else:
break
return True
Have you tried driver.implicitly_wait. It is like a setting for the driver, so you only call it once in the session and it basically tells the driver to wait the given amount of time until each command can be executed.
driver = webdriver.Chrome()
driver.implicitly_wait(10)
So if you set a wait time of 10 seconds it will execute the command as soon as possible, waiting 10 seconds before it gives up. I've used this in similar scroll-down scenarios so I don't see why it wouldn't work in your case. Hope this is helpful.
To be able to fix this answer, I have to add new text. Be sure to use a lower case 'w' in implicitly_wait.
Here I did it using a rather simple form:
from selenium import webdriver
browser = webdriver.Firefox()
browser.get("url")
searchTxt=''
while not searchTxt:
try:
searchTxt=browser.find_element_by_name('NAME OF ELEMENT')
searchTxt.send_keys("USERNAME")
except:continue
Solution for ajax pages that continuously load data. The previews methods stated do not work. What we can do instead is grab the page dom and hash it and compare old and new hash values together over a delta time.
import time
from selenium import webdriver
def page_has_loaded(driver, sleep_time = 2):
'''
Waits for page to completely load by comparing current page hash values.
'''
def get_page_hash(driver):
'''
Returns html dom hash
'''
# can find element by either 'html' tag or by the html 'root' id
dom = driver.find_element_by_tag_name('html').get_attribute('innerHTML')
# dom = driver.find_element_by_id('root').get_attribute('innerHTML')
dom_hash = hash(dom.encode('utf-8'))
return dom_hash
page_hash = 'empty'
page_hash_new = ''
# comparing old and new page DOM hash together to verify the page is fully loaded
while page_hash != page_hash_new:
page_hash = get_page_hash(driver)
time.sleep(sleep_time)
page_hash_new = get_page_hash(driver)
print('<page_has_loaded> - page not loaded')
print('<page_has_loaded> - page loaded: {}'.format(driver.current_url))
How about putting WebDriverWait in While loop and catching the exceptions.
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
browser = webdriver.Firefox()
browser.get("url")
delay = 3 # seconds
while True:
try:
WebDriverWait(browser, delay).until(EC.presence_of_element_located(browser.find_element_by_id('IdOfMyElement')))
print "Page is ready!"
break # it will break from the loop once the specific element will be present.
except TimeoutException:
print "Loading took too much time!-Try again"
You can do that very simple by this function:
def page_is_loading(driver):
while True:
x = driver.execute_script("return document.readyState")
if x == "complete":
return True
else:
yield False
and when you want do something after page loading complete,you can use:
Driver = webdriver.Firefox(options=Options, executable_path='geckodriver.exe')
Driver.get("https://www.google.com/")
while not page_is_loading(Driver):
continue
Driver.execute_script("alert('page is loaded')")
use this in code :
from selenium import webdriver
driver = webdriver.Firefox() # or Chrome()
driver.implicitly_wait(10) # seconds
driver.get("http://www.......")
or you can use this code if you are looking for a specific tag :
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Firefox() #or Chrome()
driver.get("http://www.......")
try:
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.ID, "tag_id"))
)
finally:
driver.quit()
Very good answers here. Quick example of wait for XPATH.
# wait for sizes to load - 2s timeout
try:
WebDriverWait(driver, 2).until(expected_conditions.presence_of_element_located(
(By.XPATH, "//div[#id='stockSizes']//a")))
except TimeoutException:
pass
I struggled a bit to get this working as that didn't worked for me as expected. anyone who is still struggling to get this working, may check this.
I want to wait for an element to be present on the webpage before proceeding with my manipulations.
we can use WebDriverWait(driver, 10, 1).until(), but the catch is until() expects a function which it can execute for a period of timeout provided(in our case its 10) for every 1 sec. so keeping it like below worked for me.
element_found = wait_for_element.until(lambda x: x.find_element_by_class_name("MY_ELEMENT_CLASS_NAME").is_displayed())
here is what until() do behind the scene
def until(self, method, message=''):
"""Calls the method provided with the driver as an argument until the \
return value is not False."""
screen = None
stacktrace = None
end_time = time.time() + self._timeout
while True:
try:
value = method(self._driver)
if value:
return value
except self._ignored_exceptions as exc:
screen = getattr(exc, 'screen', None)
stacktrace = getattr(exc, 'stacktrace', None)
time.sleep(self._poll)
if time.time() > end_time:
break
raise TimeoutException(message, screen, stacktrace)
If you are trying to scroll and find all items on a page. You can consider using the following. This is a combination of a few methods mentioned by others here. And it did the job for me:
while True:
try:
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
driver.implicitly_wait(30)
time.sleep(4)
elem1 = WebDriverWait(driver, 30).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "element-name")))
len_elem_1 = len(elem1)
print(f"A list Length {len_elem_1}")
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
driver.implicitly_wait(30)
time.sleep(4)
elem2 = WebDriverWait(driver, 30).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "element-name")))
len_elem_2 = len(elem2)
print(f"B list Length {len_elem_2}")
if len_elem_1 == len_elem_2:
print(f"final length = {len_elem_1}")
break
except TimeoutException:
print("Loading took too much time!")
selenium can't detect when the page is fully loaded or not, but javascript can. I suggest you try this.
from selenium.webdriver.support.ui import WebDriverWait
WebDriverWait(driver, 100).until(lambda driver: driver.execute_script('return document.readyState') == 'complete')
this will execute javascript code instead of using python, because javascript can detect when page is fully loaded, it will show 'complete'. This code means in 100 seconds, keep tryingn document.readyState until complete shows.
nono = driver.current_url
driver.find_element(By.XPATH,"//button[#value='Send']").click()
while driver.current_url == nono:
pass
print("page loaded.")

How to handle the timeout exception in python

I am writing a bot that will follow the users on www.quora.com . following is the part of code I am using where I get timeout exception:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
import urllib
driver = webdriver.Firefox()
driver.get('http://www.quora.com/')
time.sleep(10)
wait = WebDriverWait(driver, 10)
form = driver.find_element_by_class_name('regular_login')
time.sleep(10)
#add explicit wait
username = form.find_element_by_name('email')
time.sleep(10)
#add explicit wait
username.send_keys('abc#gmail.com')
time.sleep(30)
#add explicit wait
password = form.find_element_by_name('password')
time.sleep(30)
#add explicit wait
password.send_keys('def')
time.sleep(30)
#add explicit wait
password.send_keys(Keys.RETURN)
time.sleep(30)
#search = driver.find_element_by_name('search_input')
search = wait.until(EC.presence_of_element_located((By.XPATH, "//form[#name='search_form']//input[#name='search_input']")))
search.clear()
search.send_keys('Kevin Rose')
search.send_keys(Keys.RETURN)
link = wait.until(EC.presence_of_element_located((By.LINK_TEXT, "Kevin Rose")))
link.click()
#Wait till the element is loaded (Asynchronusly loaded webpage)
handle = driver.window_handles
driver.switch_to.window(handle[1])
#switch to new window
element = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.PARTIAL_LINK_TEXT, "Followers")))
time.sleep(30)
element.click()
#goes to Kevin Rose followers page
time.sleep(30)
button = driver.find_elements_by_xpath("//a[contains(text(), 'Follow')]")
#Locate follow button on the page
no_of_followers = len(button)
#total number of unfollowed users
print no_of_followers
while(no_of_followers > 0):
# execute only if there are unfollowed users on page
count = 1
while(count < no_of_followers):
time.sleep(30)
link = wait.until(EC.presence_of_element_located((By.LINK_TEXT, "Follow")))
time.sleep(30)
link.click()
time.sleep(30)
print count
count = count + 1
time.sleep(30)
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(30)
button = driver.find_elements_by_xpath("//a[contains(text(), 'Follow')]")
time.sleep(30)
no_of_followers = len(button)
After executing the code i am getting "TimeoutException" error in the inner loop after successful execution once.
How can I solve this?
Traceback:
Traceback (most recent call last): File "C:\Python27\quorabot7",
line 72, in
link = wait.until(EC.presence_of_element_located((By.LINK_TEXT, "Follow"))) File
"C:\Python27\lib\site-packages\selenium\webdriver\support\wait.py",
line 71, in until
raise TimeoutException(message) TimeoutException: Message: ''
You're getting a TimeoutException because Selenium can't find that element within the time that you've set as your wait. This means that your locator strategy is incorrect.
I have not tested your other locators, but if it is truly the inner loop that is failing... my solution is below.
After looking through the DOM on Kevin Hart's page, I can see that the button you're interested in is:
<a class="follow_button with_count" href="#" action_click="UserFollow" id="__w2_Mab4s9V_follow_user">Follow<span class="count">43.8k</span></a>
You should try this:
link = wait.until(EC.presence_of_element_located(\
(By.className, "follow_button with_count")))
or this:
link = wait.until(EC.presence_of_element_located(\
(By.XPATH, '//a[#action_click="UserFollow"]')))
The correct is By.CLASS_NAME, not By.ClassName

Categories