This is a follow-up question to this:
WebDriver element found, but click returns nothing
I am trying to scrape data from the URL in the code after making selections in the drop-down menu. I first click on Progress Monitoring and then Physical and Financial Project Summary. Then I make the following selections: State, District, Block, Year, Batch, and Collaboration. I would also like to check the Road Wise button and then click on the view button. After the table loads, I would like to click on the save button and download the excel file. In the code below I also loop through different selections under "State" item. Here is my code:
from selenium import webdriver
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
import time
import os
chromedriver = r"C:\Users\yuppal\chromedriver"
os.environ["webdriver.chrome.driver"] = chromedriver
browser = webdriver.Chrome(chromedriver)
browser.implicitly_wait(10)
browser.get("http://omms.nic.in")
browser.maximize_window()
#Click on the item Progress Monitoring
progElem = browser.find_element_by_link_text("Progress Monitoring").click()
#Click on the item Physical and Financial Project Sumamry
summElem = browser.find_element_by_link_text("Physical and Financial Project Summary").click()
#Find the element for state and create a list of different selection options
stateElem = browser.find_element_by_xpath("//select[#name='StateCode']")
state_options = stateElem.find_elements_by_tag_name("option")
#delete the first option in the list
del state_options[0]
def select_option(xpath, text):
'''
This function will select the remaining dropd-down menu items.
'''
elem = browser.find_element_by_xpath(xpath)
Select(elem).select_by_visible_text(text)
#run the loop for each option in the list of states
for option in state_options:
select_state = Select(stateElem).select_by_value(option.get_attribute("value"))
# Select the district.
select_option("//select[#name='DistrictCode']","All Districts")
# Select the block.
select_option("//select[#name='BlockCode']","All Blocks")
# Select the year.
select_option("//select[#name='Year']","All Years")
# Select the batch.
select_option("//select[#name='Batch']","All Batches")
# Select the funding agency.
select_option("//select[#name='FundingAgency']","Regular PMGSY")
# Check the road wise box.
time.sleep(10)
checkElem = WebDriverWait(browser, 120).until(EC.element_to_be_clickable((By.XPATH, "//input[#title='Road Wise']")))
browser.execute_script("arguments[0].click();", checkElem)
# Click on the view button.
time.sleep(10)
browser.find_element_by_xpath("//input[#type='button']").click()
# Switch to a new frame.
time.sleep(10)
frame = browser.find_element_by_xpath("//div[#id='loadReport']/iframe")
browser.switch_to.default_content()
#browser.switch_to.frame(frame)
WebDriverWait(browser, 120).until(EC.frame_to_be_available_and_switch_to_it(frame))
#browser.switch_to.frame(browser.find_element_by_xpath("//*[#id='loadReport']/iframe"))
# click on the save button
time.sleep(10)
WebDriverWait(browser, 120).until(EC.element_to_be_clickable((By.XPATH, "//a[#title='Export drop down menu']"))).click()
# Within the save button, Click on the "Excel" option.
time.sleep(10)
WebDriverWait(browser, 20).until(EC.element_to_be_clickable((By.XPATH, "//div/a[#title='Excel']"))).click()
# Switch back to the main content.
time.sleep(20)
browser.switch_to.default_content()
My issue is the "Road Wise" checkbox gets clicked only for some states. Thus the loop proceeds without clicking the checkbox for some states. I checked the HTML code and it is the same for all checkboxes.
I thought the problem might be that the "View" button gets clicked before the road wise button is clickable. So I put some waiting period before both road wise and view buttons. But that doesn't seem to help. So I can't really understand why the checkbox button isn't clicked for some iterations in the loop.
Before clicking on the checkbox, check that is already selected or not:
# Check the road wise box.
time.sleep(10)
checkElem = WebDriverWait(browser, 120).until(EC.element_to_be_clickable((By.XPATH, "//input[#title='Road Wise']")))
if checkElem.is_selected() != True:
browser.execute_script("arguments[0].click();", checkElem)
PS: In your case, the click will be only in the first iteration of the loop.
Related
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import Select
from selenium.webdriver.chrome import options
import unittest
import time
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
#link to website
website = 'https://www.ncei.noaa.gov/access/monitoring/climate-at-a-glance'
path = ('../chromedriver') #Folder location where is the chromedriver
driver = webdriver.Chrome(path)
driver.get(website)
driver.maximize_window()
#Selection of the sections where the information I am looking for is located
state_click = driver.find_element(by=By.XPATH, value='//*[#id="show-statewide"]/a').click()
time_series_click = driver.find_element(by=By.XPATH, value='.//*[#id="time-series"]/div[3]/button').click()
#selection of the years (for all files the same range of 1950 - 2021)
star_year_dropdown =Select(driver.find_element(by=By.ID, value='begyear'))
star_year_dropdown.select_by_visible_text('1950')
end_year_dropdown = Select(driver.find_element(by=By.ID, value='endyear'))
end_year_dropdown.select_by_visible_text('2021')
#selection of the parameter to download: Average temperature
parameter_dropdown = Select(driver.find_element(by=By.ID, value='parameter'))
parameter_dropdown.select_by_visible_text('Average Temperature')
#Creating a loop to loop through all the states and all the months:
#state selection
select_state = driver.find_element(by=By.XPATH, value='.//*[#id="state"]')
opcion_state = select_state.find_elements(by=By.TAG_NAME, value='option')
#month selection
select_month = driver.find_element(by=By.XPATH, value = '//*[#id="month"]')
opcion_month = select_month.find_elements(by = By.TAG_NAME, value='option')
for option in opcion_month:
option.click()
for option in opcion_state:
option.click()
time.sleep(3)
plot = driver.find_element(by=By.XPATH, value='.//input[#id="submit"]').click()
dowload = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//*[#id="csv-download"]'))).click()
time.sleep(3)
The code works fine, but in the plot and download functions created (at the end of the whole) when trying to click it gives an error and cannot be solved. I think it is because the web page at the time of executing that command does not display the button in the screen to plot the graph and download the csv. I have tried modifying the waiting time, but not work. Let's see if someone can help me.
Thanks in advance!!
So the problem that you are having with downloading the CSV file is not that the command is wrong (it does work), however, the issue is that the download CSV button is not visible on the page, which prevents you from clicking it.
A way to get around having to visibly see the element on the page and still click it you can do the following:
driver.execute_script("arguments[0].click();", driver.find_element(By.XPATH, '//*[#id="csv-download"]'))
This would be the preferred method, otherwise you would have to scroll to where the button is visible on the page and then click the button. The page has a scrolling effect if trying to click on a button which you can do the following (but the previous method is preferred as it is cleaner and does not take any additional time to do):
from selenium.common.exceptions import ElementClickInterceptedException
download_btn = driver.find_element(By.XPATH, '//*[#id="csv-download"]')
try:
download_btn.click() # the first time you try to click it will throw an error, while navigating the button into the page view
except ElementClickInterceptedException: # catch the exception and proceed as if clicking again
download_btn.click()
Other comments on your code
you have option.click() twice in your code, and it is unclear if you want to click the month or state option - there may be a confusion about which options are clicked. I would suggest renaming your iterator variables appropriately so that you know which buttons are being clicked
Hi Before starting Thanks for the help in advance
So I am trying to scrape google flight website : https://www.google.com/travel/flights
When scraping I have done the sending Key to the text field but I am stuck at clicking the search button it always gives the error that the field is not clickable at a point or Other elements would receive the click
the error image is
and the code is
from time import sleep
from selenium import webdriver
chromedriver_path = 'E:/chromedriver.exe'
def search(urrl):
driver = webdriver.Chrome(executable_path=chromedriver_path)
driver.get(urrl)
asd= "//div[#aria-label='Enter your destination']//div//input[#aria-label='Where else?']"
driver.find_element_by_xpath("/html/body/c-wiz[2]/div/div[2]/c-wiz/div/c-wiz/c-wiz/div[2]/div[1]/div[1]/div[1]/div[2]/div[1]/div[4]/div/div/div[1]/div/div/input").click()
sleep(2)
TextBox = driver.find_element_by_xpath(asd)
sleep(2)
TextBox.click()
sleep(2)
print(str(TextBox))
TextBox.send_keys('Karachi')
sleep(2)
search_button = driver.find_element_by_xpath('//*[#id="yDmH0d"]/c-wiz[2]/div/div[2]/c-wiz/div/c-wiz/c-wiz/div[2]/div[1]/div[1]/div[2]/div/button/div[1]')
sleep(2)
search_button.click()
print(str(search_button))
sleep(15)
print("DONE")
driver.close()
def main():
url = "https://www.google.com/travel/flights"
print(" Getitng Data ")
search(url)
if __name__ == "__main__":
main()
and i have done it by copying the Xpath using dev tools
Thanks again
The problem you are facing is that after entering the city Karachi in the text box, there is a suggestion dropdown that is displayed over the Search Button. That is the cause of the exception as the dropdown would receive the click instead of the Search button. The intended usage of the website is to select the city from the dropdown and then continue.
A quick fix would be to first look for all of the dropdowns in the source (there are a few) and look for the one that is currently active using is_displayed(). Next you would select the first element in the dropdown suggested:
.....
TextBox.send_keys('Karachi')
sleep(2)
# the attribute(role) in the dropdowns element are named 'listbox'. Warning: This could change in the future
all_dropdowns = driver.find_elements_by_css_selector('ul[role="listbox"]')
# the active dropdown
active_dropdown = [i for i in all_dropdowns if i.is_displayed()][0]
# click the first suggestion in the dropdown (Note: this is very specific to your search. It could not be desirable in other circumstances and the logic can be modified accordingly)
active_dropdown.find_element_by_tag_name('li').click()
# I recommend using the advise #cruisepandey has offered above regarding usage of relative path instead of absolute xpath
search_button = driver.find_element_by_xpath("//button[contains(.,'Search')]")
sleep(2)
search_button.click()
.....
Also suggest to head the advise provided by #cruisepandey including research more about Explicit Waits in selenium to write better performing selenium programs. All the best
Instead of this absolute xapth
//*[#id="yDmH0d"]/c-wiz[2]/div/div[2]/c-wiz/div/c-wiz/c-wiz/div[2]/div[1]/div[1]/div[2]/div/button/div[1]
I would recommend you to have a relative path:
//button[contains(.,'Search')]
Also, I would recommend you to have explicit wait when you are trying a click operation.
Code:
search_button = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//button[contains(.,'Search')]")))
search_button.click()
You'd need below imports as well:
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
Pro tip:
Launch the browser in full screen mode.
driver.maximize_window()
You should have the above line just before driver.get(urrl) command.
This is the first webpage that I've scraped, and some of the other solutions I've found don't quite seem to help. As you'll see, the "Next" button is still visible, but the CSS changes just a bit, when you get to the last page.
A few notes. I'm using python, selenium and google chrome.
I am trying to loop through each part of the table on this page: https://caearlyvoting.sos.ca.gov/
I have figured out how to loop through each county, and grab the information I need(i think). However, I am getting hung up on how to move to the next page when the table has more records than the 10 displayed by default.
I've tried variations of this
try:
next_page = driver.find_element_by_class_name('paginate_button')
next_page.click()
except NoSuchElementException:
pass
But no luck. I've tried getting the element in different ways but I run into the same issues.
Can someone help me figure out how to click through each page, grab what I need and then move onto the next county? I don't need help grabbing the info from the table, just clicking through the pages and then moving onto the next county.
EDIT
Here's the rest of the code based on a follow up. I am having difficulty structuring it.
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
import pandas as pd
import time # not for production
# Name of the counties Single column with county names
county_df = pd.read_csv('Counties.csv')
# Path to driver on this computer
chrome_driver_path = r'C:\Windows\chromedriver'
# url to scrape
url = 'https://caearlyvoting.sos.ca.gov/'
with webdriver.Chrome(executable_path=chrome_driver_path)as driver:
# Open window, maximize and set an implicit wait
driver.get(url)
driver.maximize_window()
driver.implicitly_wait(10)
actions = ActionChains(driver) #* New line here from stackoverflow
# find the county selection
county_selector = driver.find_element_by_id('CountyID')
# for loop tomove through the counties
for county in county_df['County'][:5]:
# Input the county namne
county_selector.send_keys(county)
### Code to grab data goes here
########* Code from stackoverflow ########
while True:
next_page = driver.find_element_by_css_selector(".paginate_button.next")
next_bnt_classes = next_page.get_attribute("class")
if "disabled" in next_bnt_classes:
break #last page reached, no more next pages, break the loop
else:
actions.move_to_element(next_page).perform()
time.sleep(0.5)
#get the actual next page button and click it
driver.find_element_by_css_selector(".paginate_button.next a").click()
You are using wrong locator.
Also the next page button can appear out of the view, on the bottom of the page, so you will have to scroll to that element and only after that click it.
On the last page the next page button is disabled.
In this case it contains disabled class name.
So your code can be:
from selenium.webdriver.common.action_chains import ActionChains
actions = ActionChains(driver)
while True:
#grab the data from current page, after that:
next_page = driver.find_element_by_css_selector(".paginate_button.next")
next_bnt_classes = next_page.get_attribute("class")
if "disabled" in next_bnt_classes:
break #last page reached, no more next pages, break the loop
else:
next_page = driver.find_element_by_css_selector(".paginate_button.next")
actions.move_to_element(next_page).perform()
time.sleep(0.5)
#get the actual next page button and click it
driver.find_element_by_css_selector(".paginate_button.next a").click()
UPD
The working code is slightly different:
from selenium.webdriver.common.action_chains import ActionChains
actions = ActionChains(driver)
while True:
#grab the data from current page, after that:
next_page = driver.find_element_by_css_selector(".paginate_button.next")
next_bnt_classes = next_page.get_attribute("class")
if next_bnt_classes == 'paginate_button next disabled':
break #last page reached, no more next pages, break the loop
else:
# Move to the next page for the county and append the data
next_page.click()
I want to send search term to a listbox, capture/print the url instead of clicking on it. If there is a better way than using Selenium that would also be acceptable also.
Example:
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
import time
#Choose Browser
driver=webdriver.Chrome()
time.sleep(1)
#Go to url
driver.get("https://coinmarketcap.com/coins/")
#fill out search field
driver.find_element_by_xpath("""/html/body/div/div/div[1]/div[3]/nav/nav/form/div/div/div/input""").send_keys("eth")
#Select first option from dropdown
listbox = Select(driver.find_element_by_xpath("/html/body/div/div/div[1]/div[3]/nav/nav/form/div/div[2]/ul/li[2]/a/span"))
print(listbox.select_by_index(0)) # I want to print the link instead of clicking it
I found that simply entering text in the searchbox did not show the dropdown. Here is a sample of code that will give you the innerHTML of the dropdown. You can modify the xpath to get your specific li element or parse it accordingly with BeautifulSoup.
driver.get("https://coinmarketcap.com/coins/")
#fill out search field
search_box = driver.find_element_by_xpath("""/html/body/div/div/div[1]/div[3]/nav/nav/form/div/div/div/input""")
search_box.click()
time.sleep(1)
search_box.send_keys("eth")
#Select first option from dropdown
listbox = driver.find_element_by_xpath("//div[#class='cmc-popover__dropdown']")
print(listbox.get_attribute('innerHTML'))
Edit: Also, if the driver instance is not wide enough, the search box won't appear. Consider opening it maximized.
I've written a script in python in combination with selenium to get some information from a webpage. To reach the content it is necessary to hurdle few steps, as in accepting condition, fill in the inputbox, click on the search button to populate results and finally click on the first grid (the first tr, more specifically) within the populated table. As soon as any click is initiated on the first tr, a new page (containing desired information) opens up.
My script can do the first three steps successfully. What I can't do are:
perform a click on the first tr
focus to the newly opened tab (containing information I'm after)
To reach the content:
This is the link to follow. There is an accept button to click first. Then there is an inputbox Name to be filled in with HMC DESIGN GROUP. Now, pressing the search button, the result should appear below within a table. From there I need to click on the first tr.
This is what I've tried so far:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
link = "https://officialrecords.broward.org/AcclaimWeb/search/SearchTypeName"
def get_information(driver,url):
driver.get(url)
wait.until(EC.element_to_be_clickable((By.ID, "btnButton"))).click()
wait.until(EC.presence_of_element_located((By.ID,"SearchOnName"))).send_keys("HMC DESIGN GROUP")
wait.until(EC.presence_of_element_located((By.ID, "btnSearch"))).click()
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR,".t-grid-content table tr"))).click()
if __name__ == "__main__":
driver = webdriver.Chrome()
wait = WebDriverWait(driver,10)
try:
get_information(driver,link)
finally:
driver.quit()
Currently the script neither clicks on the grid the first tr of the newly generated table nor throws any error. It quits the browser gracefully.
There is another table under the div with the same class name "t-grid-content". It appears as "Loading...". So after you submit the search you really make a click on node with selector ".t-grid-content table tr", but it just don't make a proper effect
You just need more specific selector.
Try, for instance
wait.until(EC.element_to_be_clickable((By.XPATH, "//td[contains(., 'HMC DESIGN GROUP')]")))
To switch to new window, try update your function body as
driver.get(url)
current = driver.current_window_handle
wait.until(EC.element_to_be_clickable((By.ID, "btnButton"))).click()
wait.until(EC.presence_of_element_located((By.ID,"SearchOnName"))).send_keys("HMC DESIGN GROUP")
wait.until(EC.presence_of_element_located((By.ID, "btnSearch"))).click()
wait.until(EC.element_to_be_clickable((By.XPATH, "//td[contains(., 'HMC DESIGN GROUP')]"))).click()
driver.switch_to.window([window for window in driver.window_handles if window != current][0])
Just replace this css selctor :
.t-grid-content table tr
to this :
.t-grid-content table tr.t-state-selected:first-child
for clicking on first tr.
Before clicking on first tr, store the window handle as :
window_before = driver.window_handles[0]
after clicking on first tr, store the window handle of newly opened window as:
window_after = driver.window_handles[1]
Now all you have to do is to switch the focus of your web driver to newly opened windows.Like this :
driver.switch_to_window(window_after)