from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import Select
from selenium.webdriver.chrome import options
import unittest
import time
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
#link to website
website = 'https://www.ncei.noaa.gov/access/monitoring/climate-at-a-glance'
path = ('../chromedriver') #Folder location where is the chromedriver
driver = webdriver.Chrome(path)
driver.get(website)
driver.maximize_window()
#Selection of the sections where the information I am looking for is located
state_click = driver.find_element(by=By.XPATH, value='//*[#id="show-statewide"]/a').click()
time_series_click = driver.find_element(by=By.XPATH, value='.//*[#id="time-series"]/div[3]/button').click()
#selection of the years (for all files the same range of 1950 - 2021)
star_year_dropdown =Select(driver.find_element(by=By.ID, value='begyear'))
star_year_dropdown.select_by_visible_text('1950')
end_year_dropdown = Select(driver.find_element(by=By.ID, value='endyear'))
end_year_dropdown.select_by_visible_text('2021')
#selection of the parameter to download: Average temperature
parameter_dropdown = Select(driver.find_element(by=By.ID, value='parameter'))
parameter_dropdown.select_by_visible_text('Average Temperature')
#Creating a loop to loop through all the states and all the months:
#state selection
select_state = driver.find_element(by=By.XPATH, value='.//*[#id="state"]')
opcion_state = select_state.find_elements(by=By.TAG_NAME, value='option')
#month selection
select_month = driver.find_element(by=By.XPATH, value = '//*[#id="month"]')
opcion_month = select_month.find_elements(by = By.TAG_NAME, value='option')
for option in opcion_month:
option.click()
for option in opcion_state:
option.click()
time.sleep(3)
plot = driver.find_element(by=By.XPATH, value='.//input[#id="submit"]').click()
dowload = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//*[#id="csv-download"]'))).click()
time.sleep(3)
The code works fine, but in the plot and download functions created (at the end of the whole) when trying to click it gives an error and cannot be solved. I think it is because the web page at the time of executing that command does not display the button in the screen to plot the graph and download the csv. I have tried modifying the waiting time, but not work. Let's see if someone can help me.
Thanks in advance!!
So the problem that you are having with downloading the CSV file is not that the command is wrong (it does work), however, the issue is that the download CSV button is not visible on the page, which prevents you from clicking it.
A way to get around having to visibly see the element on the page and still click it you can do the following:
driver.execute_script("arguments[0].click();", driver.find_element(By.XPATH, '//*[#id="csv-download"]'))
This would be the preferred method, otherwise you would have to scroll to where the button is visible on the page and then click the button. The page has a scrolling effect if trying to click on a button which you can do the following (but the previous method is preferred as it is cleaner and does not take any additional time to do):
from selenium.common.exceptions import ElementClickInterceptedException
download_btn = driver.find_element(By.XPATH, '//*[#id="csv-download"]')
try:
download_btn.click() # the first time you try to click it will throw an error, while navigating the button into the page view
except ElementClickInterceptedException: # catch the exception and proceed as if clicking again
download_btn.click()
Other comments on your code
you have option.click() twice in your code, and it is unclear if you want to click the month or state option - there may be a confusion about which options are clicked. I would suggest renaming your iterator variables appropriately so that you know which buttons are being clicked
Related
Hi Before starting Thanks for the help in advance
So I am trying to scrape google flight website : https://www.google.com/travel/flights
When scraping I have done the sending Key to the text field but I am stuck at clicking the search button it always gives the error that the field is not clickable at a point or Other elements would receive the click
the error image is
and the code is
from time import sleep
from selenium import webdriver
chromedriver_path = 'E:/chromedriver.exe'
def search(urrl):
driver = webdriver.Chrome(executable_path=chromedriver_path)
driver.get(urrl)
asd= "//div[#aria-label='Enter your destination']//div//input[#aria-label='Where else?']"
driver.find_element_by_xpath("/html/body/c-wiz[2]/div/div[2]/c-wiz/div/c-wiz/c-wiz/div[2]/div[1]/div[1]/div[1]/div[2]/div[1]/div[4]/div/div/div[1]/div/div/input").click()
sleep(2)
TextBox = driver.find_element_by_xpath(asd)
sleep(2)
TextBox.click()
sleep(2)
print(str(TextBox))
TextBox.send_keys('Karachi')
sleep(2)
search_button = driver.find_element_by_xpath('//*[#id="yDmH0d"]/c-wiz[2]/div/div[2]/c-wiz/div/c-wiz/c-wiz/div[2]/div[1]/div[1]/div[2]/div/button/div[1]')
sleep(2)
search_button.click()
print(str(search_button))
sleep(15)
print("DONE")
driver.close()
def main():
url = "https://www.google.com/travel/flights"
print(" Getitng Data ")
search(url)
if __name__ == "__main__":
main()
and i have done it by copying the Xpath using dev tools
Thanks again
The problem you are facing is that after entering the city Karachi in the text box, there is a suggestion dropdown that is displayed over the Search Button. That is the cause of the exception as the dropdown would receive the click instead of the Search button. The intended usage of the website is to select the city from the dropdown and then continue.
A quick fix would be to first look for all of the dropdowns in the source (there are a few) and look for the one that is currently active using is_displayed(). Next you would select the first element in the dropdown suggested:
.....
TextBox.send_keys('Karachi')
sleep(2)
# the attribute(role) in the dropdowns element are named 'listbox'. Warning: This could change in the future
all_dropdowns = driver.find_elements_by_css_selector('ul[role="listbox"]')
# the active dropdown
active_dropdown = [i for i in all_dropdowns if i.is_displayed()][0]
# click the first suggestion in the dropdown (Note: this is very specific to your search. It could not be desirable in other circumstances and the logic can be modified accordingly)
active_dropdown.find_element_by_tag_name('li').click()
# I recommend using the advise #cruisepandey has offered above regarding usage of relative path instead of absolute xpath
search_button = driver.find_element_by_xpath("//button[contains(.,'Search')]")
sleep(2)
search_button.click()
.....
Also suggest to head the advise provided by #cruisepandey including research more about Explicit Waits in selenium to write better performing selenium programs. All the best
Instead of this absolute xapth
//*[#id="yDmH0d"]/c-wiz[2]/div/div[2]/c-wiz/div/c-wiz/c-wiz/div[2]/div[1]/div[1]/div[2]/div/button/div[1]
I would recommend you to have a relative path:
//button[contains(.,'Search')]
Also, I would recommend you to have explicit wait when you are trying a click operation.
Code:
search_button = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//button[contains(.,'Search')]")))
search_button.click()
You'd need below imports as well:
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
Pro tip:
Launch the browser in full screen mode.
driver.maximize_window()
You should have the above line just before driver.get(urrl) command.
One of my dad's friends wants me to make a bot that automates buying gun primer because they are always sold out. I have a script with some ammunition as a test (because its actually in stock), but as I try and figure out how to click the dropdown, the terminal always spits out this error
selenium.common.exceptions.InvalidArgumentException: Message: invalid type: null, expected a string at line 1 column 12
For reference, I am using Python 3.9 with Selenium 3.141.0 and this is my code:
#imports funcs from selenium
from selenium import webdriver
#chooses your browser
driver = webdriver.Firefox()
#gets your store
url = driver.get("https://www.midwayusa.com/product/2090655809")
#opens the website
driver.get(url)
#makes variables for html elements
button = driver.find_element_by_xpath('//*[#id="productSelectorContainer"]/div[1]/button')
#le button click has arrived
button.click()
I think I see the issue. You have these two lines
#gets your store
url = driver.get("https://www.midwayusa.com/product/2090655809")
#opens the website
driver.get(url)
driver.get() doesn't return anything so null gets stored into url. Then on the next line you are navigating to null. These lines are actually redundant and looks like you misunderstood how to use them. You only need the first line, with some corrections.
I removed the redundant line, corrected the other, and made some adjustments to some of your comments to make them more accurate.
#imports funcs from selenium
from selenium import webdriver
#launches the browser
driver = webdriver.Firefox()
#navigates to your store website
driver.get("https://www.midwayusa.com/product/2090655809")
#stores the button in a variable
button = driver.find_element_by_xpath('//*[#id="productSelectorContainer"]/div[1]/button')
#le button click has arrived
button.click()
Faults:
url = driver.get("https://www.midwayusa.com/product/2090655809")
is not a valid just use driver.get('url')
You need to click the consent otherwise it overlap all the clicks otherwise.
Here's an example of clicking the 50 quantity of the item at hand and then proceeding.
wait = WebDriverWait(driver, 5)
driver.get("https://www.midwayusa.com/product/2090655809")
driver.maximize_window()
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR,"#cookie-consent-btn"))).click()
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR,"#productSelectorContainer > div.product-filter.original-selector > button"))).click()
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR,"#Quantity > ol > li:nth-child(1) > button"))).click()
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR,"button.big-primary-button.add-to-cart-button"))).click()
Import
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
Outputs
I am attempting to download a file from the CDC website by clicking a button in a dropdown menu (I would just access the file URL directly, but the blob URL seems to change every time the download button is clicked when checking my download history on Chrome). This button can be found by clikcing the header "Data Table for Trends in Number of COVID-19 Vaccinations in the US" below the chart on https://covid.cdc.gov/covid-data-tracker/#vaccination-trends.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
import time
driver_path = "some_path"
vac_trend_url = "https://covid.cdc.gov/covid-data-tracker/#vaccination-trends"
driver = webdriver.Chrome(driver_path)
driver.maximize_window()
driver.get(vac_trend_url)
table_title_xpath = "/html/body/div[7]/div[2]/main/div[2]/div[1]/h4"
table_title = WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.XPATH, table_title_xpath)))
table_title.click()
download_button_xpath = "/html/body/div[7]/div[2]/main/div[2]/div[2]/div[1]/button/.."
download_button = WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.XPATH, download_button_xpath)))
download_button.click()
# ActionChains(driver).move_to_element(WebDriverWait(driver, 5).until(
# EC.element_to_be_clickable((By.XPATH, download_button_xpath)))).click().perform()
Although the first click to reveal the actual download button in the dropdown works, the second click to actually download the file does not.
Message: element click intercepted: Element is not clickable at point (1445, 310)
The title explains my error running the code above. When I try to uncomment the ActionChain line, I get the error described here despite using the very same method described in the post that should fix said error. The button does not seem to be overlayed with some other HTML element, so I am even more confused about what the issue is. Is there a workaround?
EDIT: I also tried MoveTargetOutOfBoundsException problem with chromedriver version >74 at another user's suggestion, still no dice.
This should simply download the file for you. By sending click to the element.
download_button_xpath = "/html/body/div[7]/div[2]/main/div[2]/div[2]/div[1]/button"
download_button = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, download_button_xpath)))
driver.execute_script("arguments[0].click();", download_button )
Good morning,
I am trying to automatize simple online procedure: go to this site http://nvidia-research-mingyuliu.com/gaugan/
Check the little box, update a picture, render it and download it clicking the download button.
I have written my code, which is extremely intuitive and very short, but for some reasons sometimes it works and sometimes it doesn't.
Sure enough, sometimes python returns the following error:
no such element: Unable to locate element: {"method":"css selector","selector":"[id="myCheck"]"}
My code is:
import selenium
import time
from selenium import webdriver
# Using Chrome to access web
driver = webdriver.Chrome(executable_path='/Users/Marco/Downloads/chromedriver')
# Open the website
driver.get('http://nvidia-research-mingyuliu.com/gaugan/')
# Select the checkbox
time.sleep(5)
check_box = driver.find_element_by_id('myCheck')
check_box.click()
# Upload File button
element = driver.find_element_by_id("myCheck")
driver.execute_script("arguments[0].click();", element)
choose_file = driver.find_element_by_id('segmapfile')
# Send the file location to the button
choose_file.send_keys('/Users/Marco/Desktop/Foto upload/Schermata 2020-10-31 alle 00.07.03.png')
#Locate submit button and click
submit_assignment = driver.find_element_by_id('btnSegmapLoad')
submit_assignment.click()
#render
render_button = driver.find_element_by_id('render')
render_button.click()
#download
save_box = driver.find_element_by_id('save_render')
time.sleep(5)
save_box.click()
Why is this happening? How should i solve it?
Thank you
Use explicit waits which wait for something to happen, as opposed to 'time.sleep(5)' which just waits for 5 seconds:
import selenium
from selenium import webdriver
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
# Using Chrome to access web
driver = webdriver.Chrome(executable_path='/Users/Marco/Downloads/chromedriver')
# Open the website
driver.get('http://nvidia-research-mingyuliu.com/gaugan/')
# Wait for checkbox to be located
check_box_wait = EC.presence_of_element_located((By.ID, 'myCheck'))
WebDriverWait(driver, 10).until(check_box_wait)
# Select the checkbox
check_box = driver.find_element_by_id('myCheck')
check_box.click()
New to Selenium. I'm trying to get it so that I can enter a name into a search box and then click on the correct name.
I have managed to write some code to go to a website and then enter what I want and press the search button. The problem is that it then shows a list of items so I'd have to click again.
from selenium import webdriver
from selenium.webdriver.support.ui import Select
from webdriver_manager.chrome import ChromeDriverManager
from time import sleep
index = 'AMZN'
driver = webdriver.Chrome(ChromeDriverManager().install())
driver.get('https://www.marketscreener.com')
sleep(6)
text_box = driver.find_element_by_css_selector('#autocomplete')
text_box.send_keys(index)
#select.select_by_index(1)
driver.find_element_by_xpath("//*[#id='recherche_menu']/table/tbody/tr[1]/td/button/img").click()
When I enter the name and don't click It presents a dropdown list so I am trying to select the first item from the dropdown as that is usually what I will want.
I attempted this by using the commented out select.select_by_index code line. But it doesn't quite work.
I just tried also using text_box.send_keys(Keys.DOWN, Keys.RETURN) to move down into the dropdown field, but this doesn't work and just returns the same as what I currently get from clicking.
To be clear what I mean is that currently the code will return this:
But I want it to go straight to the Amazon page so it will return this:
Any help appreciated.
Thanks
This page use many JavaScript to listen the action of the mouse and the key.If you didn't focus on the searchbox,the dropdown will be disappeared.(and you couldn't find it in the source code).
Try to use ActionChains,this works fine on my PC:
from selenium import webdriver
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.action_chains import ActionChains
from time import sleep
index = 'AMZN'
driver = webdriver.Chrome()
driver.get('https://www.marketscreener.com')
action = ActionChains(driver)
sleep(4)
text_box = driver.find_element_by_css_selector('#autocomplete')
action.move_to_element(text_box)
action.click(text_box)
action.send_keys(index)
action.perform()
sleep(1)
driver.find_element_by_xpath('//*[#id="AC_tRes"]/li[1]').click()
If the result couldn't show you,maybe you need to increase the time of sleep().