We Keep Coding

click on web Scraping without getting an error with selenium

from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import Select
from selenium.webdriver.chrome import options
import unittest
import time
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
#link to website
website = 'https://www.ncei.noaa.gov/access/monitoring/climate-at-a-glance'
path = ('../chromedriver') #Folder location where is the chromedriver
driver = webdriver.Chrome(path)
driver.get(website)
driver.maximize_window()
#Selection of the sections where the information I am looking for is located
state_click = driver.find_element(by=By.XPATH, value='//*[#id="show-statewide"]/a').click()
time_series_click = driver.find_element(by=By.XPATH, value='.//*[#id="time-series"]/div[3]/button').click()
#selection of the years (for all files the same range of 1950 - 2021)
star_year_dropdown =Select(driver.find_element(by=By.ID, value='begyear'))
star_year_dropdown.select_by_visible_text('1950')
end_year_dropdown = Select(driver.find_element(by=By.ID, value='endyear'))
end_year_dropdown.select_by_visible_text('2021')
#selection of the parameter to download: Average temperature
parameter_dropdown = Select(driver.find_element(by=By.ID, value='parameter'))
parameter_dropdown.select_by_visible_text('Average Temperature')
#Creating a loop to loop through all the states and all the months:
#state selection
select_state = driver.find_element(by=By.XPATH, value='.//*[#id="state"]')
opcion_state = select_state.find_elements(by=By.TAG_NAME, value='option')
#month selection
select_month = driver.find_element(by=By.XPATH, value = '//*[#id="month"]')
opcion_month = select_month.find_elements(by = By.TAG_NAME, value='option')
for option in opcion_month:
option.click()
for option in opcion_state:
option.click()
time.sleep(3)
plot = driver.find_element(by=By.XPATH, value='.//input[#id="submit"]').click()
dowload = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//*[#id="csv-download"]'))).click()
time.sleep(3)
The code works fine, but in the plot and download functions created (at the end of the whole) when trying to click it gives an error and cannot be solved. I think it is because the web page at the time of executing that command does not display the button in the screen to plot the graph and download the csv. I have tried modifying the waiting time, but not work. Let's see if someone can help me.
Thanks in advance!!

So the problem that you are having with downloading the CSV file is not that the command is wrong (it does work), however, the issue is that the download CSV button is not visible on the page, which prevents you from clicking it.
A way to get around having to visibly see the element on the page and still click it you can do the following:
driver.execute_script("arguments[0].click();", driver.find_element(By.XPATH, '//*[#id="csv-download"]'))
This would be the preferred method, otherwise you would have to scroll to where the button is visible on the page and then click the button. The page has a scrolling effect if trying to click on a button which you can do the following (but the previous method is preferred as it is cleaner and does not take any additional time to do):
from selenium.common.exceptions import ElementClickInterceptedException
download_btn = driver.find_element(By.XPATH, '//*[#id="csv-download"]')
try:
download_btn.click() # the first time you try to click it will throw an error, while navigating the button into the page view
except ElementClickInterceptedException: # catch the exception and proceed as if clicking again
download_btn.click()
Other comments on your code
you have option.click() twice in your code, and it is unclear if you want to click the month or state option - there may be a confusion about which options are clicked. I would suggest renaming your iterator variables appropriately so that you know which buttons are being clicked

Use selenium to choose dropdown value from multiple selects that have the same xpath

I'm trying to scrape data by python from this e-commerce site
Because it requires to select the shipping location first to access the data and the 3 selects have the same xpath so I use the code below
city = browser.find_element(By.XPATH,"(//select[not(#id) and not(#class)])[1]")
citydd = Select(city)
citydd.select_by_value('01') # Hanoi
time.sleep(1)
district = browser.find_element(By.XPATH,"(//select[not(#id) and not (#class)])[2]")
districtdd = Select(district)
districtdd.select_by_value('0101') # Ba Dinh
time.sleep(1)
ward = browser.find_element(By.XPATH,"(//select[not(#id) and not (#class)])[3]")
warddd = Select(ward)
warddd.select_by_value('010104') # Cong Vi
browser.find_element(By.XPATH,"//div[text()='Xác nhận']").click() # Xac nhan
It returns me this error
NoSuchElementException: Message: no such element: Unable to locate element: {"method":"xpath","selector":"(//select[not(#id) and not(#class)])[1]"}
May I know how to bypass this situation?

There is the ability to select better xpaths. You can use a relative xpaths using the label of associated select
//label[contains(text(),'Tỉnh/Thành phố')]/following-sibling::div/select
//label[contains(text(),'Quận/Huyện')]/following-sibling::div/select
//label[contains(text(),'Phường/Xã')]/following-sibling::div/select
This is the middle one identified as unique using the above:
If you're still getting no such error with these xpaths - please ensure you include explicit or implicit waits
Selenium's default wait strategy is the "the page has loaded". Most often in modern pages, the page loads, THEN scripts run which get more data or display a modal (like the popup on the image). Those async calls cause fails as nosuchelements in selenium.
Let me know if you need more information on sycnhronisation.

This is what i have tried -
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.wait import WebDriverWait
from time import sleep
from selenium import webdriver
driver = webdriver.Chrome()
wait = WebDriverWait(driver, 10)
driver.get('https://vinmart.com/')
FirstDropDown = Select(driver.find_element_by_xpath("(//select)[1]"))
FirstDropDown.select_by_index(1)
sleep(2)
SecondDropDown = Select(driver.find_element_by_xpath("(//select)[2]"))
SecondDropDown.select_by_index(1)
sleep(2)
ThirdDropDown = Select(driver.find_element_by_xpath("(//select)[3]"))
ThirdDropDown.select_by_index(1)
I have used sleep() because it will take time to populated data in the dropdown as per pervious dropdown selection.
Please mark it as answer if it resolves your problem.

How to click the Continue button within a website using selenium in python?

I am trying to write a code that is able to auto apply on job openings on indeed.com. I have managed to reach the last stage, however, the final 2 clicks on the application form is giving me a lot of trouble. Please refer to the first page as below
Once we click on continue on the first page, for the second page I first need to scroll down a bit to reach from here...
..to here and then finally click on apply.
I am stuck on the first page, as the click function does not do anything. I have written the following code:
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
driver.get("https://in.indeed.com/jobs?q=data%20analyst&l=Delhi&vjk=5c0bd416675cf4e5")
driver.find_element_by_xpath('//*[#id="apply-button-container"]/div[1]/span[1]').click()
time.sleep(5)
frame_1 = driver.find_element_by_css_selector('iframe[title="Job application form container"')
driver.switch_to.frame(frame_1)
frame_2 = driver.find_element_by_css_selector('iframe[title="Job application form"]')
driver.switch_to.frame(frame_2)
continue_btn = driver.find_element_by_css_selector('#form-action-continue')
continue_btn.click()
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//button[#id='form-action-continue']"))).click()
driver.find_element_by_xpath('//button[#id="form-action-continue"]').click()
I have tried switching the iframes again for this step but nothing happens. Even the .click() function does not do anything.
Will appreciate some help on this.

This should go through the first click if your values are inputted.
driver.get("https://in.indeed.com/jobs?q=data%20analyst&l=Delhi&vjk=5c0bd416675cf4e5")
wait=WebDriverWait(driver, 10)
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "a.indeed-apply-button"))).click()
frame_1 = driver.find_element_by_css_selector('iframe[#title="Job application form container"')
driver.switch_to.frame(frame_1)
frame_2 = driver.find_element_by_css_selector('iframe[#title="Job application form"]')
driver.switch_to.frame(frame_2)
cont="#form-action-continue"
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, cont))).click()

My script is unable to click on a certain grid

I've written a script in python in combination with selenium to get some information from a webpage. To reach the content it is necessary to hurdle few steps, as in accepting condition, fill in the inputbox, click on the search button to populate results and finally click on the first grid (the first tr, more specifically) within the populated table. As soon as any click is initiated on the first tr, a new page (containing desired information) opens up.
My script can do the first three steps successfully. What I can't do are:
perform a click on the first tr
focus to the newly opened tab (containing information I'm after)
To reach the content:
This is the link to follow. There is an accept button to click first. Then there is an inputbox Name to be filled in with HMC DESIGN GROUP. Now, pressing the search button, the result should appear below within a table. From there I need to click on the first tr.
This is what I've tried so far:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
link = "https://officialrecords.broward.org/AcclaimWeb/search/SearchTypeName"
def get_information(driver,url):
driver.get(url)
wait.until(EC.element_to_be_clickable((By.ID, "btnButton"))).click()
wait.until(EC.presence_of_element_located((By.ID,"SearchOnName"))).send_keys("HMC DESIGN GROUP")
wait.until(EC.presence_of_element_located((By.ID, "btnSearch"))).click()
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR,".t-grid-content table tr"))).click()
if __name__ == "__main__":
driver = webdriver.Chrome()
wait = WebDriverWait(driver,10)
try:
get_information(driver,link)
finally:
driver.quit()
Currently the script neither clicks on the grid the first tr of the newly generated table nor throws any error. It quits the browser gracefully.

There is another table under the div with the same class name "t-grid-content". It appears as "Loading...". So after you submit the search you really make a click on node with selector ".t-grid-content table tr", but it just don't make a proper effect
You just need more specific selector.
Try, for instance
wait.until(EC.element_to_be_clickable((By.XPATH, "//td[contains(., 'HMC DESIGN GROUP')]")))
To switch to new window, try update your function body as
driver.get(url)
current = driver.current_window_handle
wait.until(EC.element_to_be_clickable((By.ID, "btnButton"))).click()
wait.until(EC.presence_of_element_located((By.ID,"SearchOnName"))).send_keys("HMC DESIGN GROUP")
wait.until(EC.presence_of_element_located((By.ID, "btnSearch"))).click()
wait.until(EC.element_to_be_clickable((By.XPATH, "//td[contains(., 'HMC DESIGN GROUP')]"))).click()
driver.switch_to.window([window for window in driver.window_handles if window != current][0])

Just replace this css selctor :
.t-grid-content table tr
to this :
.t-grid-content table tr.t-state-selected:first-child
for clicking on first tr.
Before clicking on first tr, store the window handle as :
window_before = driver.window_handles[0]
after clicking on first tr, store the window handle of newly opened window as:
window_after = driver.window_handles[1]
Now all you have to do is to switch the focus of your web driver to newly opened windows.Like this :
driver.switch_to_window(window_after)

Clicking is not working in python selenium

I am trying to create a bot which will fill up the signup forms for a particular website.
Website Details - http://hacknyu.org/signup
Code:
from selenium import webdriver
class HackNNYU(object):
first_name = 'input[ng-model="credentials.first_name"]'
last_name = 'input[ng-model="credentials.last_name"]'
email = '.col-sm-12>input[ng-model="credentials.email"]'
password = '.col-sm-12>input[ng-model="credentials.password"]'
agree_checkbox = '.ng-binding>input[ng-model="checkModel"]'
sign_up_button = 'div>button[type="submit"]'
accept_button = 'button[ng-click="positive()"]'
def fill_up_hack_nyu():
driver = webdriver.Firefox()
driver.get('http://hacknyu.org/signup')
driver.find_element_by_css_selector(HackNNYU.first_name).send_keys('Friday')
driver.find_element_by_css_selector(HackNNYU.last_name).send_keys('Night')
driver.execute_script("window.scrollTo(0, 300);")
driver.find_element_by_css_selector(HackNNYU.email).send_keys('ade347#gmail.edu')
driver.find_element_by_css_selector(HackNNYU.password).send_keys('123456')
driver.find_element_by_css_selector(HackNNYU.agree_checkbox).click()
driver.find_element_by_css_selector(HackNNYU.accept_button).click()
# driver.execute_script("window.scrollTo(0, 400);")
driver.find_element_by_css_selector(HackNNYU.sign_up_button).click()
fill_up_hack_nyu()
Problem
driver.find_element_by_css_selector(HackNNYU.sign_up_button).click()
The main problem is in this line. Manually when you click on signup button it is working fine but when I run this program, I can see it is clicking on signup button but nothing is happening after that. Could anyone help me why this is not working? I am just trying to register for an event using a bot. I will highly appreciate any help you can provide.
Error
Sometimes I always receive this error
selenium.common.exceptions.WebDriverException: Message: Element is not clickable at point (796.4000244140625, 45.399993896484375). Other element would receive the click

The page has a banner at the top which makes the automatic scrolling hide the submit button. To overcome this issue, you can define the scrolling behaviour to scroll at the bottom instead. Moreover it seems that your script is no correctly clicking on the checkbox that should display the terms popup.
Here is a working script to create a new account on hacknyu:
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
# set the scrolling behavior to down
DesiredCapabilities.FIREFOX["elementScrollBehavior"] = 1
driver = webdriver.Firefox()
wait = WebDriverWait(driver, 10)
# load the page
driver.get("http://hacknyu.org/signup")
# get the form element
form = driver.find_element_by_css_selector("form[name='signupForm']")
# fill the fields
form.find_element_by_css_selector("input[name='firstName']").send_keys("myfirstname")
form.find_element_by_css_selector("input[name='lastName']").send_keys("mylastname")
form.find_element_by_css_selector("input[name='email']").send_keys("na#na.na")
form.find_element_by_css_selector("input[name='password']").send_keys("mypassword")
# click and accept terms
form.find_element_by_xpath("//input[#name='terms']/..").click()
wait.until(EC.presence_of_element_located((By.XPATH, "//button[.='Accept']"))).click()
wait.until_not(EC.presence_of_element_located((By.CSS_SELECTOR, ".modal")))
# click on submit
form.find_element_by_css_selector("button[type='submit']").click()

It is probably timing issue. You can use explicit wait to make sure the element is clickable
wait = WebDriverWait(driver, 10)
wait.until(expected_conditions.element_to_be_clickable((By.CSS_SELECTOR, HackNNYU.sign_up_button))).click()
This will wait up to 10 seconds for the button to be clickable before clicking on it.

Mostly providing wait or sleep before click will helps you simulate click correctly. Some situations click by using Actions also helps me. In Java
Actions moveAndClick=new Actions(driver);
moveAndClick.moveToElement(driver.findElement(By.cssSelector("HackNNYU.sign_up_button"))).click().build().perform();
and one more way is to use javascript executor.
WebElement element = driver.findElement(By.cssSelector("HackNNYU.sign_up_button"));
JavascriptExecutor executor = (JavascriptExecutor)driver;
executor.executeScript("arguments[0].click();", element);
Thank You,
Murali