Page has a spinner (when page loads)
<div id="event-wait-msg-main" style="display: block;">
<div id="event-wait-msg"></div>
</div>
When page is loaded style attribute changes to display: none;
How can I use webdriver wait in this case?
You can use WebDriverWait to wait until element will be invisible:
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
wait = WebDriverWait(driver, 10)
wait.until(EC.invisibility_of_element_located((By.ID, "event-wait-msg-main")))
You can create custom expected condition
wait = WebDriverWait(driver, 10)
wait.until(__WaitForAttributeToBe(By.id('event-wait-msg'), 'style', 'display: none;'))
class __WaitForAttributeToBe(object):
def __init__(self, by, attribute, expected_value):
self.__by = by
self.__attribute = attribute
self.__expected_value = expected_value
def __call__(self, driver):
try:
attribute_value = driver.find_element(self.__by).get_attribute(self.__attribute)
return attribute_value == self.__expected_value
except (NoSuchElementException, StaleElementReferenceException):
return False
Use explicit wait :
wait = WebDriverWait(driver, 10)
wait.until(EC.presence_of_element_located((By.xpath, './/div[contains(#style,'none')]')))
Related
driver = webdriver.Chrome()
URL= ['https://makemyhomevn.com/collections/ghe-an-cafe/products/ghe-go-tron']
driver.get(URL)
sleep(1)
des = driver.find_element_by_xpath('//div[#class="product-item-description"]//strong/following sibling::text()[1]')
print(des)
I expect my result as 'Gỗ tự nhiên', I have tried many ways but couldn't get the text after 'Chất liệu:'.
You can take the entire span text using .get_attribute('innerText') and then use the split function from Python like below:
driver.maximize_window()
wait = WebDriverWait(driver, 20)
driver.get("https://makemyhomevn.com/collections/ghe-an-cafe/products/ghe-go-tron")
time.sleep(1)
entire_span = wait.until(EC.visibility_of_element_located((By.XPATH, "//strong[text()='Chất liệu:']/..")))
entire_span_splitted = entire_span.get_attribute('innerText').split(":")
#print(entire_span_splitted[0])
print(entire_span_splitted[1])
Imports:
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
Output:
Gỗ tự nhiên.
Goal: use Python to scrape the value of 225 from the person score web form field; this field allows for user input and I think is dynamically populated by javascript?
Picture of Form:
HTML:
<input type="text" size="20" autocomplete="off" id="ext-comp-1011" name="Lead Score" class="x-form-text x-form-field x-form-num-field" title="" style="width: 189px;">
Javascript that generates form values?
My Code:
import requests
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
target_url = ['www.website_login.com','www.website_I_want_second_page.com']
driver = webdriver.Chrome("C:\\Users\\user\\Desktop\\chrome_driver\\chromedriver_win32\\chromedriver.exe")
driver.maximize_window()
driver.get(target_url[0])
driver.find_element_by_id("loginUsername").send_keys("login")
driver.find_element_by_id("loginPassword").send_keys("password")
driver.find_element_by_id("loginButton").click()
driver.get(target_url[1])
lead_detail_page = driver.page_source
soup = BeautifulSoup(lead_detail_page, 'html.parser')
form_item_ids = []
for tag in soup.findAll(class_ = "x-form-item"):
form_item_ids.append(tag.get("id"))
for i in form_item_ids:
test = soup.find(id=i).get_text()
print(test)
Result of Code:
There is a possibility, that person score value is added by javascript after retrieving page source using lead_detail_page = driver.page_source. It would make sense to try retrieve its value using selenium WebDriverWait, waiting for value of the input element being not empty:
from selenium import webdriver
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
# delay for waiting in seconds
DELAY = 30
# helper class which allows to wait for inputs value to be not empty
class wait_for_value_not_empty(object):
def __init__(self, locator, attribute = 'value', value = ''):
self.locator = locator
self.attribute = attribute
self.value = value
def __call__(self, driver):
try:
element_attribute = EC._find_element(driver, self.locator).get_attribute(self.attribute)
return element_attribute is not None and element_attribute != self.value
except StaleElementReferenceException:
return False
# wait for value to be filled
WebDriverWait(driver, DELAY).until(wait_for_value_not_empty((By.ID, "ext-comp-1011")))
# retrieve value
person_score = driver.find_element_by_id("ext-comp-1011").get_attribute('value')
print(person_score)
Hi I've tired to find the right selenium code to get click the main parent class if the following requirements exist in the class :
Parent Class
<div class ="col-xs-2-4 shopee-search-item-result__item" data-sqe="item">
Child class
<a data-sqe="link" href= all urls that is printed in python.>
Child class contains this element
<div class = "_1gkBDw _2O43P5">
<div class = "_1HvBLA">
<div class = "_3ao649" data-sqe="ad"> Ad</div>
Here is the code bellow
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
import time
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
import csv
import time
url = 'https://shopee.com.my/search?keyword=mattress'
driver = webdriver.Chrome(executable_path=r'E:/users/Francabicon/Desktop/Bots/others/chromedriver.exe')
driver.get(url)
time.sleep(0.8)
# select language
driver.find_element_by_xpath('//div[#class="language-selection__list"]/button').click()
time.sleep(3)
# scroll few times to load all items
for x in range(10):
driver.execute_script("window.scrollBy(0,300)")
time.sleep(0.1)
# get all links (without clicking)
all_items = driver.find_elements_by_xpath('//a[#data-sqe="link"]')
print('len:', len(all_items))
all_urls = []
j = 0
k = 45
for item in all_items:
url = item.get_attribute('href')
all_urls.append(url)
print(all_urls)
a= len(all_urls)
# now use links
i = 0
while i <= 4 :
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//div[#class='col-xs-2-4 shopee-search-item-result__item' and #data-sqe='item']//a[#class='link' and #href= all_urls[i]]"))).click()
i+=1
I've tried to locate:
-Div the whole class
-locate classes and the href individualy
-click the first five columns
but it all always fails.
Traceback (most recent call last):
File "E:/Users/Asashin/Desktop/Bots/click test 7.py", line 52, in <module>
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//div[#class='col-xs-2-4 shopee-search-item-result__item' and #data-sqe='item']//a[#class='link' and #href= all_urls[i]]"))).click()
File "C:\Users\User\AppData\Local\Programs\Python\Python37-32\lib\site-packages\selenium\webdriver\support\wait.py", line 80, in until
raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message:
Can I be solved?
I have made couple of changes.
When you are fetching href values you are getting complete url and not the url you are seeing in DOM so you need to remove the preceding values in order to verify later.
In the last while loop all_urls[i] is variable you need passed it as variable not string.
Once you click each link you need to come back to the parent page again by using driver.back()
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
import time
url = 'https://shopee.com.my/search?keyword=mattress'
driver = webdriver.Chrome(executable_path=r'E:/users/Francabicon/Desktop/Bots/others/chromedriver.exe')
driver.get(url)
# select language
WebDriverWait(driver,5).until(EC.element_to_be_clickable((By.XPATH,'//div[#class="language-selection__list"]/button'))).click()
time.sleep(3)
# scroll few times to load all items
for x in range(10):
driver.execute_script("window.scrollBy(0,300)")
time.sleep(0.1)
# get all links (without clicking)
all_items = driver.find_elements_by_xpath('//a[#data-sqe="link"]')
print('len:', len(all_items))
all_urls = []
j = 0
k = 45
for item in all_items:
# This give you whole url of the anchor tag
url = item.get_attribute('href')
# You need to remove the preceding values in order to verify href later for clicking
urlfinal=url.split('https://shopee.com.my')[1]
all_urls.append(urlfinal)
print(all_urls)
a= len(all_urls)
# now use links
i = 0
while i <= 4 :
#Identify the parent tag by child tag use following Xpath.
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//div[#class='col-xs-2-4 shopee-search-item-result__item' and #data-sqe='item'][.//a[#data-sqe='link' and #href='" + all_urls[i] +"']]"))).click()
driver.back()
i+=1
How to get parent element:
child_element = driver.find_element_by_xpath('//a[#data-sqe="link"]')
parent_element = child_element.find_element_by_xpath('./ancestor::div[contains(#class, "shopee-search-item-result__item")][1]')
How to get element with specific child:
element = driver.find_element_by_xpath('div[contains(#class, "shopee-search-item-result__item") and .//a[#data-sqe="link"]]')
I'm trying to automate a form assertion with a list of data, however I'm struggling on when and how to use "WebDriverWait" or driver implict wait. My list is 1000 strings. When I run a sample of a 100, less than 100 are captured correctly. The code below catches ElementNotSelectableException\StaleElementReferenceException but doesn't address them correctly.
import unittest
from selenium.common.exceptions import ElementNotVisibleException, ElementNotSelectableException, \
ElementNotInteractableException, StaleElementReferenceException
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.ui import WebDriverWait
driver = webdriver.Chrome()
oar_order_id = '#oar-order-id'
oar_zip_code = '#oar_zip'
#order confirmation list
order_id_list = ["WO-34284975","WO-50002804","WO-50004302","WO-34282964","WO-34214963"]
#zip code confirmation list
zip_code_list = ["23508","99338","62036","75074-7520","37763","98034","89406-4361"]
submit_button = 'body.sales-guest-form.page-layout-1column:nth-child(2) div.page-wrapper:nth-child(10) main.page-main:nth-child(4) div.columns:nth-child(4) div.column.main form.form.form-orders-search:nth-child(4) div.actions-toolbar div.primary > button.action.submit.primary'
# Enter orderid & Zip & click enter
found = []
notfound = []
length = len(order_id_list)
driver.implicitly_wait(10)
wait = WebDriverWait(driver, 15, poll_frequency=1,
ignored_exceptions=[ElementNotVisibleException ,ElementNotSelectableException])
driver.get('https:/www.ecklers.com/sales/orderlookup/index/')
# Sample of 10 from list
for i in range(10):
try:
wait
element1 = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, oar_order_id)))
element1.send_keys(order_id_list[i])
element2 = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, oar_zip_code)))
element2.send_keys(zip_code_list[i])
element3 = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, submit_button)))
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, submit_button)))
element3.click()
wait
wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,"body.sales-guest-view.page-layout-1column:nth-child(2) div.page-wrapper:nth-child(10) main.page-main:nth-child(4) div.page-title-wrapper:nth-child(3) h1.page-title > span.base")))
title = driver.title
#driver.implicitly_wait(60)
except(StaleElementReferenceException):
print("StaleElementReferenceException")
except(ElementNotInteractableException):
print("ElementNotInteractableException")
try:
text = order_id_list[i]
assert(title.endswith(text[3:]))
found.append(text)
except(AssertionError):
notfound.append((order_id_list[i]))
driver.get('https:/www.ecklers.com/sales/orderlookup/index/')
wait
print(found,notfound)```
I found an answer that helped me. I used: time.sleep(.500) after driver.get('https:/www.ecklers.com/sales/orderlookup/index/'
Q:
I'm using Selenium to get a page with contents, and after I click the more button,the page outputs more content,and how I get the new page through webdriver?
some codes like this:
def parase_questions(self):
driver = self.login()
driver.implicitly_wait(2)
more_btn = driver.find_element_by_css_selector(".zg-btn-white.zg-r3px.zu-button-more")
more_btn.click()
# should I do something to get the new driver ?
print driver.page_source
question_links = driver.find_elements_by_css_selector('.question_link')
print len(question_links)
If I understand you correctly, after you click the More button, there are more elements with question_link class loaded. You would need a way to wait for the question links to be loaded.
Here is one idea - a custom Expected Condition that would help you to wait until there are more than N number of elements:
from selenium.common.exceptions import StaleElementReferenceException
from selenium.webdriver.support import expected_conditions as EC
class wait_for_more_than_n_elements(object):
def __init__(self, locator, count):
self.locator = locator
self.count = count
def __call__(self, driver):
try:
count = len(EC._find_elements(driver, self.locator))
return count > self.count
except StaleElementReferenceException:
return False
Usage:
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
driver = self.login()
driver.implicitly_wait(2)
question_links = driver.find_elements_by_css_selector('.question_link')
more_btn = driver.find_element_by_css_selector(".zg-btn-white.zg-r3px.zu-button-more")
more_btn.click()
# wait
wait = WebDriverWait(driver, 10)
wait.until(wait_for_more_than_n_elements((By.CSS_SELECTOR, ".question_link"), len(question_links))
# now more question links were loaded, get the page source
print(driver.page_source)