My code goes into this website, and clicks on each row of a table.
These rows open a window, which contain a field "keywords" at the bottom of the new window, which I am trying to get.
I dont think I have the xpath for this field right, though all I did was right click and "copy xpath".
Expected output is to print the value in the keywords.
from selenium import webdriver
from bs4 import BeautifulSoup
import time
import requests
driver = webdriver.Chrome()
driver.get('https://aaaai.planion.com/Web.User/SearchSessions?ACCOUNT=AAAAI&CONF=AM2021&USERPID=PUBLIC&ssoOverride=OFF')
time.sleep(3)
page_source = driver.page_source
soup = BeautifulSoup(page_source,'html.parser')
eachRow=driver.find_elements_by_class_name('clickdiv')
for item in eachRow:
item.click() #opens the new window per each row
time.sleep(2)
faculty = driver.find_element_by_xpath("//td[#valign='MIDDLE']/b") #this works fine
keywords=item.find_element_by_xpath('//*[#id="W1"]/div/div/div/div[2]/div[2]/table/tbody/tr[5]/td/text()') #this does not
print(keywords.text)
print(faculty.text)
driver.find_element_by_class_name('XX').click()#closes window
ERROR message - selenium.common.exceptions.InvalidSelectorException: Message: invalid selector: The result of the xpath expression "//*[#id="W1"]/div/div/div/div[2]/div[2]/table/tbody/tr[5]/td/text()" is: [object Text]. It should be an element.
.....
Your XPath '//*[#id="W1"]/div/div/div/div[2]/div[2]/table/tbody/tr[5]/td/text()' is getting the actual text of the element, not the entire element itself. Just remove text() and it should work. XPath is notoriously finnicky, so be careful how you use it, and how you are defining your top level root.
I reworked your code very slightly here to give a better option than time.sleep(), explicit waits. It isn't necessary, but tends to make more reliable code and speeds up processing; it doesn't pause the process unless it has to, and only does so for as long as necessary.
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
driver = webdriver.Chrome()
driver.get('https://aaaai.planion.com/Web.User/SearchSessions?ACCOUNT=AAAAI&CONF=AM2021&USERPID=PUBLIC&ssoOverride=OFF')
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, 'clickdiv')))
eachRow = driver.find_elements_by_class_name('clickdiv')
for item in eachRow:
item.click() # opens the new window per each row
faculty = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//td[#valign='MIDDLE']/b")))
keywords = item.find_element_by_xpath('//*[#id="W1"]/div/div/div/div[2]/div[2]/table/tbody/tr[5]/td')
print(keywords.text)
print(faculty.text)
driver.find_element_by_class_name('XX').click() # closes window
Related
The following code is not writing any partial string in the From input field on the website even though this element seems to be an active element.
I spent lot of time trying to debug and make the code work but no success. Can anyone please provide some hint on what is wrong. Thanks.
from selenium import webdriver
from selenium.webdriver.common.by import By
import time
from selenium.webdriver.common.keys import Keys
from colorama import init, Fore
class BookingTest1():
def __init__(self):
pass
def test1(self):
baseUrl="https://www.goibibo.com/"
driver=webdriver.Chrome()
driver.maximize_window()
#open airline site
driver.get(baseUrl)
driver.implicitly_wait(3)
# Enter origin location.
partialTextOrigin="New"
#select flight tab
driver. find_element(By.XPATH,"//ul[#class='happy-nav']//li//a[#href='/flights/']").click()
# select input box
textElement = driver.find_element(By.XPATH, "//input")
# check if input box is active
if textElement==driver.switch_to.active_element:
print('element is in focus')
textElement.send_keys(partialTextOrigin)
else:
print('element is not in focus')
print("Focus Event Triggered")
driver.execute_script("arguments[0].focus();", textElement)
time.sleep(5)
if textElement==driver.switch_to.active_element:
print('finally element is in focus')
print(partialTextOrigin)
textElement.send_keys(partialTextOrigin)
time.sleep(5)
#test the code
tst=BookingTest1()
tst.test1()
There are several issues here:
First you need to click on p element in the From block and only after that when input appears there you can insert the text to it.
You should use unique locators. (There more that 10 input elements on this page)
Using WebDriverWait expected conditions explicit waits are much better than implicitly_wait in most cases.
No need to set timeouts to too short values.
No need to use driver.switch_to.active_element here.
The following code works for me:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(service=webdriver_service, options=options)
url = "https://www.goibibo.com/"
flights_xpath = "//ul[#class='happy-nav']//li//a[#href='/flights/']"
from_xpath = "//div[./span[contains(.,'From')]]//p[contains(text(),'Enter city')]"
from_input_xpath = "//div[./span[contains(.,'From')]]//input"
partialTextOrigin = "New"
wait = WebDriverWait(driver, 10)
driver.get(url)
wait.until(EC.element_to_be_clickable((By.XPATH, flights_xpath))).click()
wait.until(EC.element_to_be_clickable((By.XPATH, from_xpath))).click()
wait.until(EC.element_to_be_clickable((By.XPATH, from_input_xpath))).send_keys(partialTextOrigin)
from_xpath and from_input_xpath XPath locators are a little complex.
I was not sure about the class names in that elements block if they are fixed so I based on the texts.
For example "//div[./span[contains(.,'From')]]//p[contains(text(),'Enter city')]" means:
Find such div that it has a direct span child so that span contains From text content.
From the div parent element above find inside it a p child that contains Enter city text.
Similarly to the above locator "//div[./span[contains(.,'From')]]//input" means: find parent div as described before, then find inside it an input child element.
The result of the code above is
I've tried using getText() Referencing, but when I use that method I always get the error "WebElement object has no attribute getText()". I found .text through a YouTube video but I'm not getting results from it either. Any thoughts?
from selenium import webdriver
driverPath = 'C:\Program Files (x86)\chromedriver.exe'
driver = webdriver.Chrome(driverPath)
driver.implicitly_wait(20)
driver.get ('https://shop.pricechopper.com/search?search_term=lettuce')
pageSource = driver.page_source
search = driver.find_elements_by_class_name ("css-dr3s47")
print (search[0].text)
driver.quit()
You are using Selenium with Python, so correct method is .text
.getText() is for Selenium-Java Bindings.
The reason why are you getting null is cause elements are not rendered properly and you are trying to interact with them resulting in null.
Please use Explicit wait or time.sleep() which is worst case to get rid of this issue.
Also, there's a pop window that we will have to click on close web element, so that css-dr3s47 will be available to Selenium view port.
Code :
driver = webdriver.Chrome(driver_path)
driver.maximize_window()
driver.implicitly_wait(30)
wait = WebDriverWait(driver, 30)
driver.get("https://shop.pricechopper.com/search?search_term=lettuce")
try:
wait.until(EC.element_to_be_clickable((By.XPATH, "//button[#ng-click='processModalCloseClick()']"))).click()
print('Clicked on modal pop up closer button')
except:
pass
search = driver.find_elements_by_class_name ("css-dr3s47")
print (search[0].text)
Imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
getText returns elements with rendered size more than 0 * 0 , so if element is not displayed of has size zero , then it wont return anything .
Two things you could do is :
scroll to the element before getting text
Actions actionProvider = new Actions(driver);
// Performs mouse move action onto the element
actionProvider.moveToElement(search[0]).build().perform();
print(search[0].text)
Use textCOntent
Text content doesn't check for display or size , it just gives content
print(search[0].get_attribute("textContent"))
I'm practicing trying to scrape my university's course catalog. I have a few lines in Python that open the url in Chrome and clicks the search button to bring up the course catalog. When I go to extract the texting using find_elements_by_xpath(), it returns blank. When I use the dev tools on Chrome, there definitely is text there.
from selenium import webdriver
import time
driver = webdriver.Chrome()
url = 'https://courses.osu.edu/psp/csosuct/EMPLOYEE/PUB/c/COMMUNITY_ACCESS.OSR_CAT_SRCH.GBL?'
driver.get(url)
time.sleep(3)
iframe = driver.find_element_by_id('ptifrmtgtframe')
driver.switch_to.frame(iframe)
element = driver.find_element_by_xpath('//*[#id="OSR_CAT_SRCH_WK_BUTTON1"]')
element.click()
course = driver.find_elements_by_xpath('//*[#id="OSR_CAT_SRCH_OSR_CRSE_HEADER$0"]')
print(course)
I'm trying to extract the text from the element 'OSU_CAT_SRCH_OSR_CRSE_HEADER'. I don't understand why it's not returning the text values especially when I can see that it contains text with dev tools.
You are not using text that is the reason you are not getting the text.
course = driver.find_elements_by_xpath('//*[#id="OSR_CAT_SRCH_OSR_CRSE_HEADER$0"]').text
Try above changes in last second line
Below is the full code after the changes
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
driver = webdriver.Chrome()
url = 'https://courses.osu.edu/psp/csosuct/EMPLOYEE/PUB/c/COMMUNITY_ACCESS.OSR_CAT_SRCH.GBL?'
driver.get(url)
time.sleep(3)
iframe = driver.find_element_by_id('ptifrmtgtframe')
driver.switch_to.frame(iframe)
element = driver.find_element_by_xpath('//*[#id="OSR_CAT_SRCH_WK_BUTTON1"]')
element.click()
# wait 10 seconds
course = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.XPATH, '//*[#id="OSR_CAT_SRCH_OSR_CRSE_HEADER$0"]'))
).text
print(course)
So I'm scraping using selenium and I want to click 'next' button in 'Defensive' section but the code I wrote clicks 'next' on 'Summary'.
Here's the url for you to try :
https://www.whoscored.com/Regions/252/Tournaments/2/Seasons/7361/Stages/16368/PlayerStatistics/England-Premier-League-2018-2019
So it's selecting 'Defensive' and I can see it selected in the window but the next page doesnt appear. On clicking 'Summary' I found out next function is actually happening there.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
browser= webdriver.Chrome(executable_path ="C:\Program Files (x86)\Google\Chrome\chromedriver.exe")
browser.get('https://www.whoscored.com/Regions/252/Tournaments/2/Seasons/7361/Stages/16368/PlayerStatistics/England-Premier-League-2018-2019')
browser.find_element_by_xpath("""//*[#id="stage-top-player-stats-options"]/li[2]/a""").click()
element = WebDriverWait(browser, 20).until(EC.presence_of_element_located((By.XPATH, """//*[#id="next"]""")))
browser.execute_script("arguments[0].click();", element)
The xpath for next button is not unique for this page. try this,
element = WebDriverWait(browser, 20).until(EC.presence_of_element_located((By.XPATH, "//*[#id='stage-top-player-stats-defensive']//a[#id='next']")))
browser.execute_script("arguments[0].click();", element)
or
element = WebDriverWait(browser, 20).until(EC.presence_of_element_located((By.XPATH, "//*[#id='stage-top-player-stats-defensive']//a[#id='next']")))
element.click()
For each tab (Summary, Defensive, ..) new next button with same id=next added to the DOM.
Select Defensive and you will see there will be two next buttons with same id=next, select Offensive and there will be three next buttons.
With basic id=next selector you always click to the first next button from Summary tab. Because you're using JavaScript and nothing happen, try to click with Selenium click method and you will get an error.
To solve the problem adjust your selector to be more specific to the dom - #statistics-paging-defensive #next.
Also when you first time open the page there's cookies acceptance screen appears and block the page, you can use method like below to skip it.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import selenium.common.exceptions as EX
def accept_cookies():
try:
WebDriverWait(browser, 20)\
.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button.qc-cmp-button")))\
.click()
except EX.NoSuchElementException or EX.TimeoutException:
pass
#...
browser = webdriver.Chrome(executable_path ="C:\Program Files (x86)\Google\Chrome\chromedriver.exe")
browser.get('https://www.whoscored.com/Regions/252/Tournaments/2/Seasons/7361/Stages/16368/PlayerStatistics/England-Premier-League-2018-2019')
wait = WebDriverWait(browser, 20)
browser.get(baseUrl)
accept_cookies()
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "[href='#stage-top-player-stats-defensive']"))).click()
next_button = wait.until(
EC.element_to_be_clickable((By.CSS_SELECTOR, "#statistics-paging-defensive #next")))
next_button.click()
Your elements locators must be unique
Avoid using XPath wildcards - * as it will cause performance degradation and prolonged elements lookup timings
Avoid using JavaScriptExecutor for clicking, well-behaved Selenium test must do what real user does and I doubt that real user will be opening browser console and typing something like document.getElementById('next').click(), he will use the mouse
Assuming all above you should come up with a selector which uniquely identifies next button on Defensive tab which would be something like:
//div[#id='statistics-paging-defensive']/descendant::a[#id='next']
References:
XPath Tutorial
XPath Axes
XPath Operators & Functions
import webbrowser
from selenium import webdriver
browser = webdriver.Chrome()
browser.get('https://www.suntrust.com/')
browser.implicitly_wait(10)
elem = browser.find_element_by_xpath('//*[#id="sign-on-3A69E29D-79E0-403E-
9352-5261239ADD89-user"]')
elem.send_keys('your-username')
I'm having two problems:
1) The window doesn't open up in full screen, meaning the username field isn't physically visible. How do I open the url in a new tab instead of a new window.
2) Other posts suggest that the element is faked by JavaScript so that webdriver can't see it.
I've tried find_element_by in all the other locators.
Your question should be answered by a simple line of code that you need to include
browser.maximize_window()
would maximise your window. Another option is to set a specific window size like
driver.set_window_size(1280, 1024)
You can use both to achieve the browser being open to a maximum size.
Another point that I would make is that, if you're a beginner, try using more of CSS Selectors instead of the Xpath. They are much faster than Xpath's. Please see a detailed post on SQA about what makes a good locator.
For your case, the CSS Selector for the sign in field would be
driver.find_element_by_css_selector('input#sign-on-3A69E29D-79E0-403E-9352-5261239ADD89-user')
For password, it would be
driver.find_element_by_css_selector('input#sign-on-3A69E29D-79E0-403E-9352-5261239ADD89-password')
For Sign On button it would be
driver.find_element_by_css_selector('button.suntrust-login-button')
Please read more about CSS Selectors and try using them more often in your code.
Here is the Answer to your Question:
The xpath you have constructed is not unique. The xpath matches exactly to 2 elements on the HTML DOM. So Selenium was trying to send_keys on the first matching element which was invisible. Hence the error element not visible. The xpath used in the following code block identifies the User ID field uniquely and sends the text:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
options = Options()
options.add_argument("start-maximized")
options.add_argument("disable-infobars")
options.add_argument("--disable-extensions")
browser = webdriver.Chrome(chrome_options=options, executable_path="C:\\Utility\\BrowserDrivers\\chromedriver.exe")
browser.get('https://www.suntrust.com/')
browser.implicitly_wait(15)
elem = browser.find_element_by_xpath('//section[#role="main"]//input[#id="sign-on-3A69E29D-79E0-403E-9352-5261239ADD89-user"]')
elem.send_keys('your-username')
Let me know if this Answers your Question.
If you use absolute xpath then you can send the text in textbox.
Below code will do that
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
browser = webdriver.Chrome()
browser.maximize_window() # to open full size window
browser.get('https://www.suntrust.com/')
# browser.implicitly_wait(10)
WebDriverWait(browser, 10).until(
EC.presence_of_element_located((By.XPATH, '//*[#id="sign-on-3A69E29D-79E0-403E-9352-5261239ADD89-user"]')))
elem = browser.find_element_by_xpath("//div[#id='suntrust-login-form-herosignon']/div[2]/form/div[1]/input[1]")
elem.send_keys('your-username')
elem1 = browser.find_element_by_xpath("//div[#id='suntrust-login-form-herosignon']/div[2]/form/div[2]/input[1]")
elem1.send_keys('your-password')