I am trying to input the name of the city that I want to depart from into Google Flights using python. After locating the element (the input box) in the html code I noticed that once you interact with the element it changes its class from class="II2One j0Ppje zmMKJ LbIaRd" to class="II2One j0Ppje zmMKJ LbIaRd VfPpkd-ksKsZd-mWPk3d-OWXEXe-AHe6Kc-XpnDCe". Therefore driver.find_element(By.CSS_SELECTOR, '.LbIaRd').send_keys(city_name) only manages to send the first character of the string before the input box changes class. However I am unable to send any text to this element after it changes its class.
I have tried interacting with the element in order to change its class and then applying the same method to the new class of the input box:
x = driver.find_elements(By.CSS_SELECTOR, '.LbIaRd')[0]
x.click()
time.sleep(2)
driver.find_element(By.CSS_SELECTOR, '.VfPpkd-ksKsZd-mWPk3d-OWXEXe-AHe6Kc-XpnDCe').send_keys(city_name)
The code ends with no errors but it the text is not sent to the input box. There is only one element with .VfPpkd-ksKsZd-mWPk3d-OWXEXe-AHe6Kc-XpnDCe locator.
I have also tried to click the element after the class change and then send the keys:
x = driver.find_elements(By.CSS_SELECTOR, '.LbIaRd')[0]
x.click()
time.sleep(2)
driver.find_element(By.CSS_SELECTOR, '.VfPpkd-ksKsZd-mWPk3d-OWXEXe-AHe6Kc-XpnDCe').click()
driver.find_element(By.CSS_SELECTOR, '.VfPpkd-ksKsZd-mWPk3d-OWXEXe-AHe6Kc-XpnDCe').send_keys(city_name)
But I get the following exception:
selenium.common.exceptions.ElementClickInterceptedException:
Message: element click intercepted:
Element <input class="II2One j0Ppje zmMKJ LbIaRd VfPpkd-ksKsZd-mWPk3d-OWXEXe-AHe6Kc-XpnDCe, redacted html code">
is not clickable at point (612, 472).
Other element would receive the click: <input class="II2One j0Ppje zmMKJ LbIaRd", redacted html code>
Any help would be greatly appreciated.
Something like this? Try to send keys with ActionChains instead with send_keys function
# Needed libs
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
import time
# We create the driver
driver = webdriver.Chrome()
# We instantiate the ActionChains which will allow to us to send several keys
action = ActionChains(driver)
# We maximize the window
driver.maximize_window()
# We navigate to the url
url='https://www.google.com/travel/flights'
driver.get(url)
departure = "Berlin, Germany"
destiny = "Austin, Texas, USA"
# We click on Reject cookies button from pop up
WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH, "(//button)[1]"))).click()
# We clear the departure input in case there is something
WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH, "(//input)[1]"))).clear()
# We click on it
WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH, "(//input)[1]"))).click()
# We send the keys with the actionChains previously initialized
action.send_keys(departure).perform()
# We click on the first result
WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH, f"//ul[#role='listbox']//div[text()='{departure}']"))).click()
# Same for destiny input
WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH, "(//input)[3]"))).click()
action.send_keys(destiny).perform()
WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH, f"//ul[#role='listbox']//div[text()='{destiny}']"))).click()
# Click on search button
WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH, f"//button/span[text()='Search']"))).click()
Related
I'm trying to click on all the links on a web page after clicking on the dates (https://www.eduqas.co.uk/qualifications/computer-science-as-a-level/#tab_pastpapers) but the links don't have unique class names and only have a tag name "a" but multiple other elements have the same tag name. How can I click on the links
Here is the current code, it clicks on the dates but as I said I can't click on the links:
from selenium import webdriver
from selenium.webdriver.common.by import By
from time import sleep
PATH = "C:\Program Files (x86)\chromedriver.exe" # path of chrome driver
driver = webdriver.Chrome(PATH) # accesses the chrome driver
driver.get("https://www.eduqas.co.uk/qualifications/computer-science-as-a-level/#tab_pastpapers") # website
driver.maximize_window()
driver.implicitly_wait(3)
driver.execute_script("window.scrollTo(0, 540)")
sleep(3) # Giving time to fully load the content
elements = driver.find_elements(By.CSS_SELECTOR, ".css-13punl2")
driver.find_element(By.ID, 'accept-cookies').click() # Closes the cookies prompt
for x in elements:
if x.text == 'GCSE':
continue
x.click()
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);") # this scrolls the page to the bottom
sleep(1) # This sleep is necessary to give time to finish scrolling
print(len(elements))
Image of links
I think you are trying this:
# Needed libs
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
import time
driver = webdriver.Chrome()
driver.get('https://www.eduqas.co.uk/qualifications/computer-science-as-a-level/#tab_pastpapers')
driver.maximize_window()
actions = ActionChains(driver)
# We get how many dates (year) fields we have
dates_count = WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.XPATH, "//div[#id='pastpapers_content']//button[#class='css-13punl2']/..")))
driver.find_element(By.ID, 'accept-cookies').click()
# We do scroll to the year element and we click every year
for i in range(1, len(dates_count)+1):
date = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, f"(//div[#id='pastpapers_content']//button[#class='css-13punl2']/..)[1]")))
driver.execute_script("arguments[0].scrollIntoView();", date)
time.sleep(0.3)
date.click()
time.sleep(0.3)
# For every year we get all the links
links = date = WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.XPATH, f"(//div[#class='css-1301qtx'])[{i}]//a")))
# We do scroll to the link element and we click every link
for link in links:
driver.execute_script("arguments[0].scrollIntoView();", link)
time.sleep(0.3)
link.click()
driver.switch_to.window(driver.window_handles[0])
This open every singlle link in the page.
I hope the comments into the code help to understand what the code does
The main reason for using XPath is when you don’t have a suitable id or name attribute for the element you wish to locate. You can use XPath to either locate the element in absolute terms (not advised) or relative to an element that does have an id or name attribute. XPath locators can also be used to specify elements via attributes other than id and name.
Find the elements:
elem = driver.find_element(By.XPATH, "/html/body/form[1]")
Get XPath:
Go to the inspect window
Select the desired element and right-click on it
In the dropdown click on copy tab, then on copy XPath
Hope this helps.Happy Coding:)
I am writing a webscraping script that automatically logs into my Email account and sends a message.
I have written the code to the point where the browser has to input the message. I don't know how to access the input field correctly. I have seen that it is an iframe element. Do I have to use the switch_to_frame() method and how can I do that? How can I switch to the iframe if there is no name attribute? Do I need the switch_to_frame() method or can I just use the find_element_by_css_selector() method?
This is the source code of the iframe:
Here is my code:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
myPassword = 'xxxxxxxxxxxxxxxx'
browser = webdriver.Firefox() # Opens Firefox webbrowser
browser.get('https://protonmail.com/') # Go to protonmail website
loginButton = WebDriverWait(browser, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, "a.btn-ghost:nth-child(1)")))
loginButton.click()
usernameElem = WebDriverWait(browser, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "#username")))
usernameElem.send_keys("first.last#protonmail.com")
passwordElem = browser.find_element_by_css_selector("#password")
passwordElem.send_keys(myPassword)
anmeldenButton = browser.find_element_by_css_selector(".button")
anmeldenButton.click()
newMessage = WebDriverWait(browser, 10).until(EC.element_to_be_clickable((By.XPATH, "/html/body/div[1]/div[3]/div/div/div[1]/div[2]/button")))
newMessage.click()
addressElem = WebDriverWait(browser, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "input[id^='to-composer']")))
addressElem.send_keys('first.last#mail.com')
subjectElem = WebDriverWait(browser, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "input[id^='subject-composer']")))
subjectElem.send_keys('anySubject')
messageElem = WebDriverWait(browser, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "#squire > div > div:nth-child(1)")))
messageElem.send_keys('message')
To access the <input> field within the iframe so you have to:
Induce WebDriverWait for the desired frame to be available and switch to it.
Induce WebDriverWait for the desired element to be clickable.
You can use either of the following Locator Strategies:
Using CSS_SELECTOR:
WebDriverWait(driver, 20).until(EC.frame_to_be_available_and_switch_to_it((By.CSS_SELECTOR,"iframe[title='Editor']")))
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div#squire"))).send_keys('message')
Using XPATH:
WebDriverWait(driver, 20).until(EC.frame_to_be_available_and_switch_to_it((By.XPATH,"//iframe[#title='Editor']")))
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//div[#id='squire']"))).send_keys('message')
Note : You have to add the following imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
PS: As the <div> tag is having the attribute contenteditable="true" you can still send text to the element.
Reference
You can find a couple of relevant discussions in:
Switch to an iframe through Selenium and python
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element while trying to click Next button with selenium
selenium in python : NoSuchElementException: Message: no such element: Unable to locate element
You have to switch to iframe with driver.switch_to.frame method.
Like any other web element iframe element can be located by ID, CLASS, XPATH, CSS_SELECTOR etc.
Looks like here you can use this method:
WebDriverWait(driver, 10).until(EC.frame_to_be_available_and_switch_to_it((By.CSS_SELECTOR,"iframe[title='Editor']")))
Or
WebDriverWait(driver, 10).until(EC.frame_to_be_available_and_switch_to_it((By.CSS_SELECTOR,"iframe[data-testid='squire-iframe']")))
When finished working within the iframe you will have to switch back to the default content with
driver.switch_to.default_content()
you first need to switch to iframe
wait = WebDriverWait(driver, 30)
wait.until(EC.frame_to_be_available_and_switch_to_it((By.CSS_SELECTOR, "iframe[title='Editor']")))
and now here write the code to send the message body. something like this:
wait.until(EC.frame_to_be_available_and_switch_to_it((By.CSS_SELECTOR, "iframe[title='Editor']")))
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div#squire"))).click()
email = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "div#squire div:nth-child(2)")))
email.send_keys('write the email here')
also once you are done with iframe interaction, you should switch to default content:
driver.switch_to.default_content()
I've been trying to scrape this website Link I'm interested in clicking the first DONWLOAD button. However whenever I try to find any element that is a button, I can't find any.
Here is the code :
url = 'https://ember-climate.org/data/carbon-price-viewer/'
webdriver = create_driver()
with webdriver as driver:
driver.maximize_window()
driver.get(url)
wait = WebDriverWait(driver, 30)
try:
wait.until(EC.element_to_be_clickable((By.XPATH, '//button')))
except:
pass
ids = driver.find_elements_by_xpath('//button')
for ii in ids:
print (ii.tag_name, ii.get_attribute('class'))
Is there anything wrong with the XPath or is it an issue with the website itself?
Your xpath is wrong, Download button is wrapped inside an span tag not button tag.
try this instead :
//span[contains(text(),'DOWNLOAD')]
also, I see it's in iframe, which can be located via
iframe[name='ETS']
CSS_SELECTOR, and we need to switch also to this iframe.
so in sequence the explanation would be :
You would have to click on cookies button.
Download button is in an iframe, we need to switch to iframe first and then we can interact with download button.
download button is a part of span tag not button tag.
Use Explicit waits.
Prefer id, css over xpath. (if they are unique in nature)
Launch browser in full screen mode.
Code :
driver = webdriver.Chrome(driver_path)
driver.maximize_window()
#driver.implicitly_wait(30)
wait = WebDriverWait(driver, 50)
driver.get("https://ember-climate.org/data/carbon-price-viewer/")
wait.until(EC.element_to_be_clickable((By.ID, "cn-accept-cookie"))).click()
wait.until(EC.frame_to_be_available_and_switch_to_it((By.CSS_SELECTOR, "iframe[name='ETS']")))
wait.until(EC.element_to_be_clickable((By.XPATH, "//span[contains(text(),'DOWNLOAD')]"))).click()
Imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
The following Selenium automated script correctly opens the given URL and opens the View Invoice tab that opens up a detailed Invoice.
I need to fetch some values like Number, Date and table values from the detailed invoice. The values are very nested to get to them correctly. The URL that opens up when the View Invoice is clicked, I don't know how to scrape it or use selenium to proceed with.
Is the element in the code like an instance to get the values of the opened detailed invoice page or is there some different approach?
Here is the code:
from selenium import webdriver
driver = webdriver.Chrome(executable_path=r'D:/Chrome driver/chromedriver.exe') # Get local session(use webdriver.Chrome() for chrome)
driver.implicitly_wait(3)
driver.get("URL") # load page from some url
driver.find_element_by_xpath("//input[#id='PNRId']").send_keys("MHUISZ")
driver.find_element_by_xpath("//input[#id='GstRetrievePageInteraction']").click()
element = driver.find_element_by_name('ViewInvoice')
element.click()
Can anyone please guide me on how to fetch the values from the invoice page?
So try to wait for elements to be visible or clickable and your clicking on the invoice actually creates new child handles so you have to switch to them. All you have to do now is figure how to go through a table try looking through it's xpath.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Chrome(executable_path=r'D:/Chrome driver/chromedriver.exe') # Get local session(use webdriver.Chrome() for chrome)
driver.get("URL")
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//input[#id='PNRId']"))).send_keys("MHUISZ")
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, "//input[#id='GstRetrievePageInteraction']"))).click()
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.NAME, 'ViewInvoice'))).click()
p = driver.current_window_handle
#get first child window
chwnd = driver.window_handles
for w in chwnd:
#switch focus to child window
if(w!=p):
driver.switch_to.window(w)
break
invoiceTable = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, "TableHeader")))
print(invoiceTable.find_element_by_xpath("tbody/tr[1]/td").text)
driver.quit()
So I'm scraping using selenium and I want to click 'next' button in 'Defensive' section but the code I wrote clicks 'next' on 'Summary'.
Here's the url for you to try :
https://www.whoscored.com/Regions/252/Tournaments/2/Seasons/7361/Stages/16368/PlayerStatistics/England-Premier-League-2018-2019
So it's selecting 'Defensive' and I can see it selected in the window but the next page doesnt appear. On clicking 'Summary' I found out next function is actually happening there.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
browser= webdriver.Chrome(executable_path ="C:\Program Files (x86)\Google\Chrome\chromedriver.exe")
browser.get('https://www.whoscored.com/Regions/252/Tournaments/2/Seasons/7361/Stages/16368/PlayerStatistics/England-Premier-League-2018-2019')
browser.find_element_by_xpath("""//*[#id="stage-top-player-stats-options"]/li[2]/a""").click()
element = WebDriverWait(browser, 20).until(EC.presence_of_element_located((By.XPATH, """//*[#id="next"]""")))
browser.execute_script("arguments[0].click();", element)
The xpath for next button is not unique for this page. try this,
element = WebDriverWait(browser, 20).until(EC.presence_of_element_located((By.XPATH, "//*[#id='stage-top-player-stats-defensive']//a[#id='next']")))
browser.execute_script("arguments[0].click();", element)
or
element = WebDriverWait(browser, 20).until(EC.presence_of_element_located((By.XPATH, "//*[#id='stage-top-player-stats-defensive']//a[#id='next']")))
element.click()
For each tab (Summary, Defensive, ..) new next button with same id=next added to the DOM.
Select Defensive and you will see there will be two next buttons with same id=next, select Offensive and there will be three next buttons.
With basic id=next selector you always click to the first next button from Summary tab. Because you're using JavaScript and nothing happen, try to click with Selenium click method and you will get an error.
To solve the problem adjust your selector to be more specific to the dom - #statistics-paging-defensive #next.
Also when you first time open the page there's cookies acceptance screen appears and block the page, you can use method like below to skip it.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import selenium.common.exceptions as EX
def accept_cookies():
try:
WebDriverWait(browser, 20)\
.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button.qc-cmp-button")))\
.click()
except EX.NoSuchElementException or EX.TimeoutException:
pass
#...
browser = webdriver.Chrome(executable_path ="C:\Program Files (x86)\Google\Chrome\chromedriver.exe")
browser.get('https://www.whoscored.com/Regions/252/Tournaments/2/Seasons/7361/Stages/16368/PlayerStatistics/England-Premier-League-2018-2019')
wait = WebDriverWait(browser, 20)
browser.get(baseUrl)
accept_cookies()
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "[href='#stage-top-player-stats-defensive']"))).click()
next_button = wait.until(
EC.element_to_be_clickable((By.CSS_SELECTOR, "#statistics-paging-defensive #next")))
next_button.click()
Your elements locators must be unique
Avoid using XPath wildcards - * as it will cause performance degradation and prolonged elements lookup timings
Avoid using JavaScriptExecutor for clicking, well-behaved Selenium test must do what real user does and I doubt that real user will be opening browser console and typing something like document.getElementById('next').click(), he will use the mouse
Assuming all above you should come up with a selector which uniquely identifies next button on Defensive tab which would be something like:
//div[#id='statistics-paging-defensive']/descendant::a[#id='next']
References:
XPath Tutorial
XPath Axes
XPath Operators & Functions