I couldn't find a way to automate the date picker using Selenium.
from selenium import webdriver
from getpass import getpass
import pandas as pd
import numpy as np
import requests
import lxml
url = "https://www.skyscanner.ca"
driver = webdriver.Chrome("chromedriver")
driver.maximize_window()
driver.get(url)
trip_type_id = "fsc-trip-type-selector-one-way"
trip_type_select = driver.find_element(by="id", value=trip_type_id )
trip_type_select.click()
origin_textbox = driver.find_element(by="id", value="fsc-origin-search" )
origin_value = "Vancouver (Any)"
destin_textbox = driver.find_element(by="id", value="fsc-destination-search")
dest_value = "Dubai (Any)"
destin_textbox.send_keys(dest_value)
I am stuck at the datepicker
driver.find_element(by="id", value="depart-fsc-datepicker-button" ).click()
opens the datepicker as
enter image description here
Is there a way to automate by passing a specific date ("December 12, 2022") with selenium?.
After analyzing the DOM of the page I found that the calendar class has dropdown for month_year and is available in the option tab. The element can be selected either on the basis of text or on the basis class name. I have used text here.
Similarly the date part can be selected by selecting the span on text which is present inside the button.
Here is the code that selects the given date on the page.
import re
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
# REPLACE YOUR CHROME PATH HERE
chrome_path = r"C:\Users\hpoddar\Desktop\Tools\chromedriver_win32\chromedriver.exe"
s = Service(chrome_path)
driver = webdriver.Chrome(service=s)
driver.get('https://www.skyscanner.ca')
driver.maximize_window()
date = 'December 12, 2022' # ENTER YOUR DATE HERE
# Extracting day and month_year from the date
m = re.search(r'([A-Za-z]+) (\d{2}), (\d{4})', date)
day, month_year = m.group(2), m.group(1) + " " + m.group(3)
driver.find_element(by="id", value="depart-fsc-datepicker-button" ).click()
click_on_depart_dropdown = driver.find_element(by="id", value="depart-calendar__bpk_calendar_nav_select" )
click_on_depart_dropdown.click()
monthyear = click_on_depart_dropdown.find_element(By.XPATH, f'//option[contains(text(), "{month_year}")]')
monthyear.click()
day_element = click_on_depart_dropdown.find_element(By.XPATH, f'//button[contains(#class, "BpkCalendarDate_bpk-calendar-date__MTdlO")]//span[contains(text(), "{day}")]')
day_element.click()
The above example was for Depart, similarly it can be done for Return.
Related
I am trying to select some data from webpage: https://indiawris.gov.in/wris/#/groundWater
I am facing problem while picking the start and end dates in the webpage. I have made an attempt using a code similar to this earlier, but here it did not work.
#code snippet tried for date picking earlier which is not working now
startdate_ele = driver.find_element_by_id('start_date_1')
startdate_ele.clear()
startdate = '2016/11/01' ## set the start date
end_date_ele = driver.find_element_by_id('end_date_1')
end_date_ele.clear()
endDate ='2017/07/31' ## set the end date for files to download
startdate_ele.send_keys(startdate)
time.sleep(5)
end_date_ele.send_keys(endDate)
Following is the current code.
#functional code that does selection prior to the date picking
from selenium.webdriver.support.ui import WebDriverWait
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium import webdriver
import time
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import Select
path2 = r"F:\\chromedriver_win32\\chromedriver.exe"
driver = webdriver.Chrome(executable_path=path2)
driver.maximize_window()
wait = WebDriverWait(driver, 30)
driver.get("https://indiawris.gov.in/wris/#/groundWater")
# try:
wait.until(EC.frame_to_be_available_and_switch_to_it((By.XPATH, "//iframe[#class='ng-star-inserted']")))
print('Switched successfully to iframe')
ele = wait.until(EC.element_to_be_clickable((By.XPATH, "//div[#class='views']//label[contains(.,'Basin')]//input")))
driver.execute_script("arguments[0].click();", ele)
print('Clicked on basin button')
sourceDrop = driver.find_element_by_xpath('//*[#id="applications"]/div[2]/div/groundwater-sidebar/div/div[2]')
sourceItems = sourceDrop.find_elements_by_class_name('ng-star-inserted')
print("length",len(sourceItems))
actionChains = ActionChains(driver)
select = Select(driver.find_element_by_xpath('//*[#id="applications"]/div[2]/div/groundwater-sidebar/div/div[2]/select'))
select.select_by_visible_text('CGWB')
select = Select(driver.find_element_by_xpath('//*[#id="applications"]/div[2]/div/groundwater-sidebar/div/div[3]/select'))
select.select_by_visible_text('yyyyMMdd')
startdate_ele = driver.find_element_by_xpath('//*[#id="applications"]/div[2]/div/groundwater-sidebar/div/div[4]/datepicker/div/input')
startdate_ele.clear()
startdate = '03-January-2021' ## set the start date
end_date_ele = driver.find_element_by_id('end_date_1')
end_date_ele.clear()
endDate ='2017/07/31' ## set the end date for files to download
startdate_ele.send_keys(startdate)
time.sleep(5)
end_date_ele.send_keys(endDate)
can some one help me in picking the dates from this portal to do the selection.
There are several issues currently wrong with your code.
1. Use By.XPATH, By,CLASS_NAME, By.ID all of your items are depreciated.
2. select = Select(driver.find_element(By.XPATH,'//*[#id="applications"]/div[2]/div/groundwater-sidebar/div/div[3]/select'))
select.select_by_value('yyyyMMdd')
Using the wrong selection for the timestep
3. Target the input tag and send keys for start and stop they have something like
<input _ngcontent-qlb-c14="" bsdatepicker="" class="form-control ng-pristine ng-valid ng-star-inserted ng-touched" readonly="" ng-reflect-klass="form-control" ng-reflect-ng-class="[object Object]" ng-reflect-bs-config="[object Object]" ng-reflect-is-disabled="true" ng-reflect-max-date="Sat Jan 09 2021 00:00:00 GMT-0" ng-reflect-name="to" placeholder="Select to date" style="">
For selecting Months below code will work to navigate w.r.t year we need to add more logic.
eleq = wait.until(EC.element_to_be_clickable((By.XPATH,"(//span[#class='input-group-addon pointer'])[1]")))
driver.execute_script("arguments[0].click();", eleq)
driver.find_element_by_xpath("//span[contains(text(),'January')]").click()
#eleq = wait.until(EC.element_to_be_clickable((By.XPATH,"//span[#class='input-group-addon pointer']")))
eleqy = wait.until(EC.element_to_be_clickable((By.XPATH,"(//span[#class='input-group-addon pointer'])[2]")))
driver.execute_script("arguments[0].click();", eleqy)
driver.find_element_by_xpath("//span[contains(text(),'January')]").click()
I am trying to scrape data from a website using Selenium, I am able to send values but not receiving the result to start scraping. The program is also not throwing any errors. Here's the reproducible code:
# Setting up driver
driver = webdriver.Chrome(executable_path='D:\Program Files\ChromeDriver\chromedriver.exe')
# Opening up the webpage
driver.get("https://www1.nseindia.com/products/content/derivatives/equities/historical_fo.htm")
# Setting the value of Select Instrument
driver.find_element_by_id('instrumentType').send_keys('Index Options')
# Setting the value of Select Symbol
driver.find_element_by_id('symbol').send_keys('NIFTY 50')
# Setting the value of Select Year
driver.find_element_by_id('year').send_keys('2019')
# Setting the value of Select Expiry
select = Select(driver.find_element_by_id('expiryDate'))
noOfExpiries = 2
select.select_by_index(noOfExpiries)
# Setting the value of Select Option Type
cycle = 'PE'
driver.find_element_by_id('optionType').send_keys(cycle)
# Clicking the date range radio button
driver.find_element_by_xpath("//input[#id='rdDateToDate']").click()
# Setting the date range
fromDate = (datetime.strptime(select.options[noOfExpiries].text, '%d-%m-%Y') - timedelta(days=45)).strftime("%d-%b-%Y")
driver.find_element_by_id('fromDate').send_keys(fromDate)
toDate = datetime.strptime(select.options[noOfExpiries].text, '%d-%m-%Y').strftime("%d-%b-%Y")
driver.find_element_by_id('toDate').send_keys(toDate)
print(fromDate, toDate)
# Clicking the Get Data button
driver.find_element_by_id('getButton').click()
Any clue as to what am I missing here?
Okay, this should do it. I came up with selenium based solution only because you expressed disinterest in requests module:
from selenium import webdriver
from datetime import datetime, timedelta
from selenium.webdriver.common.by import By
from selenium.webdriver import ChromeOptions
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
options = ChromeOptions()
options.add_argument('disable-blink-features=AutomationControlled')
with webdriver.Chrome(options=options) as driver:
wait = WebDriverWait(driver,10)
driver.get("https://www1.nseindia.com/products/content/derivatives/equities/historical_fo.htm")
wait.until(EC.presence_of_element_located((By.ID, "instrumentType"))).send_keys('Index Options')
wait.until(EC.presence_of_element_located((By.ID, "symbol"))).send_keys('NIFTY 50')
wait.until(EC.presence_of_element_located((By.ID, "year"))).send_keys('2019')
select = Select(wait.until(EC.presence_of_element_located((By.ID, "expiryDate"))))
noOfExpiries = 13
select.select_by_index(noOfExpiries)
cycle = 'PE'
wait.until(EC.presence_of_element_located((By.ID, "optionType"))).send_keys(cycle)
item = wait.until(EC.presence_of_element_located((By.XPATH, "//input[#id='rdDateToDate']")))
driver.execute_script("arguments[0].click();",item)
fromDate = (datetime.strptime(select.options[noOfExpiries].text, '%d-%m-%Y') - timedelta(days=45)).strftime("%d-%b-%Y")
wait.until(EC.presence_of_element_located((By.ID, "fromDate"))).send_keys(fromDate)
toDate = datetime.strptime(select.options[noOfExpiries].text, '%d-%m-%Y').strftime("%d-%b-%Y")
wait.until(EC.presence_of_element_located((By.ID, "toDate"))).send_keys(toDate,Keys.ENTER)
print(fromDate, toDate)
search_button = wait.until(EC.presence_of_element_located((By.ID, "getButton")))
driver.execute_script("arguments[0].click();",search_button)
tabular_content = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "#historicalData > table"))).get_attribute("innerHTML")
print(tabular_content)
I am not able to print the link of the final pdf which is opening after running the given code
from selenium import webdriver
from selenium.webdriver.support import ui
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
def page_is_loaded(driver):
return driver.find_element_by_tag_name("body")!= None
def check_exists_by_text(text):
try:
driver.find_element_by_link_text(text)
except NoSuchElementException:
return False
return True
driver = webdriver.Chrome("C:/Users/Roshan/Desktop/sbi/chromedriver")
driver.maximize_window()
driver.get("http://www.careratings.com/brief-rationale.aspx")
wait = ui.WebDriverWait(driver,10)
wait.until(page_is_loaded)
location_field = driver.find_element_by_name("txtfromdate")
location_field.send_keys("2019-05-06")
last_date = driver.find_element_by_name("txttodate")
last_date.send_keys("2019-05-21")
driver.find_element_by_xpath("//input[#name='btn_submit']").click()
if check_exists_by_text('Reliance Capital Limited'):
elm =driver.find_element_by_link_text('Reliance Capital Limited')
driver.implicitly_wait(5)
elm.click()
driver.implicitly_wait(50)
#time.sleep(5)
#driver.quit()
else :
print("Company is not rated in the given Date range")
I am expecting the actual output is the link of this pdf :
"http://www.careratings.com/upload/CompanyFiles/PR/Reliance%20Capital%20Ltd.-05-18-2019.pdf"
but I do not know how to print this link
You need to find all elements in table, then extract data from them.
from selenium import webdriver
import os
# setup path to chrome driver
chrome_driver = os.getcwd() + '/chromedriver'
# initialise chrome driver
browser = webdriver.Chrome(chrome_driver)
# load url
browser.get('http://www.careratings.com/brief-rationale.aspx')
# setup date range
location_field = browser.find_element_by_name("txtfromdate")
location_field.send_keys("2019-05-06")
last_date = browser.find_element_by_name("txttodate")
last_date.send_keys("2019-05-21")
browser.find_element_by_xpath("//input[#name='btn_submit']").click()
# get all data rows
content = browser.find_elements_by_xpath('//*[#id="divManagementSpeak"]/table/tbody/tr/td/a')
# get text and href link from each element
collected_data = []
for item in content:
url = item.get_attribute("href")
description = item.get_attribute("innerText")
collected_data.append((url, description ))
Output:
('http://www.careratings.com/upload/CompanyFiles/PR/Ashwini%20Frozen%20Foods-05-21-2019.pdf', 'Ashwini Frozen Foods')
('http://www.careratings.com/upload/CompanyFiles/PR/Vanita%20Cold%20Storage-05-21-2019.pdf', 'Vanita Cold Storage')
and so on
I would say you just need to put this line:
pdf_link = elm.get_attribute("href")
Just check out the below image. You have missed one important part to click on. When you enter some text in that inputbox, there is a dropdown projected downward displaying the search results available in their stock to choose from. Once you click on that, the rest are as it is.
Try the following script:
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
url = "http://www.careratings.com/brief-rationale.aspx"
with webdriver.Chrome() as driver:
driver.get(url)
wait = WebDriverWait(driver,10)
location_field = wait.until(EC.presence_of_element_located((By.NAME, "txtfromdate")))
location_field.send_keys("2019-05-06")
last_date = wait.until(EC.presence_of_element_located((By.NAME, "txttodate")))
last_date.send_keys("2019-05-21")
input_search = wait.until(EC.presence_of_element_located((By.NAME, "txtSearchCompany_brief")))
input_search.send_keys('Reliance Capital Limited')
time.sleep(3) #could not get rid of this hardcoded delay to make the script work
wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,"[onclick*='Reliance Capital Limited']"))).click()
# time.sleep(2) #activate this line in case the script behaves otherwise
wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,"input[name='btn_submit']"))).click()
for item in wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR,"table tr td > a[href$='.pdf']"))):
print(item.get_attribute("href"))
I need to search for a flight with python selenium but I couldn't able to select my desirable date.
import time
import selenium
from selenium import webdriver
browser = webdriver.Chrome()
browser.get("https://www.spicejet.com/")
departureButton = browser.find_element_by_id("ctl00_mainContent_ddl_originStation1_CTXT")
departureButton.click()
browser.find_element_by_partial_link_text("Kolkata").click()
arivalButton = browser.find_element_by_id("ctl00_mainContent_ddl_destinationStation1_CTXT")
arivalButton.click()
browser.find_element_by_partial_link_text("Goa").click()
date_position = browser.find_element_by_id("ctl00_mainContent_view_date1")
date_position.click()
search_date = "10-September 2019"
dep_date1 = search_date.split("-")
dep_month = dep_date[1]
dep_day = dep_date[0]
cal_head = browser.find_elements_by_class_name("ui-datepicker-title")
for month_hd in cal_head:
month_year = month_hd.text
if dep_month == month_year:
pass
else:
nxt = browser.find_element_by_class_name("ui-icon-circle-triangle-e").click()
print(month_year)
time.sleep(2)
browser.close()
The problem with your code is that when you click on the next button the DOM changes and the element reference saved in your variables are not updated. That is why it gives you stale element reference exception. Instead of using variables, use the locator for every time you access the calendar elements and it will work.
Try This :
import time
import selenium
from selenium import webdriver
browser = webdriver.Chrome()
browser.get("https://www.spicejet.com/")
departureButton = browser.find_element_by_id("ctl00_mainContent_ddl_originStation1_CTXT")
departureButton.click()
browser.find_element_by_partial_link_text("Kolkata").click()
arivalButton = browser.find_element_by_id("ctl00_mainContent_ddl_destinationStation1_CTXT")
arivalButton.click()
browser.find_element_by_partial_link_text("Goa").click()
date_position = browser.find_element_by_id("ctl00_mainContent_view_date1")
date_position.click()
search_date = "10-September 2019"
dep_date = search_date.split("-")
dep_month = dep_date[1]
dep_day = dep_date[0]
while browser.find_element_by_class_name("ui-datepicker-title").text != dep_month:
browser.find_element_by_css_selector("a[data-handler='next']").click()
browser.find_element_by_xpath("//table//a[text()='"+dep_day+"']").click()
time.sleep(2)
browser.close()
I am scraping this website :
https://login.aviva.com.sg/directinsurance/homeinsurance.htm
I want to fill all the form elements from a csv file (we will call it "profil". When I try to fill the form with a unique "profil" everything is working fine. But when I do my loop on the different profiles I have multiple problems:
sometimes I can't "get" the full address from the postalcode (see below) and so I can't get the final quotation
the driver is refreshing but I don't have the quote from the different people in profil.
Here is a type of profil that I have:
profil = [["MRS ","Corinne","SIMON","F","M","600 ","No, for myself and my family","72603190","2017-H1","CO ","Ridout Road","10","91 - 124","27 - 38","099197","S4553141D","1958","5","1"],
["MS ","Corinne","MOREAU","F","D","610 ","Yes, for myself","63856280","2017-H1","CO ","Stevens","10","38 - 208","24 - 40","099198","S9186686B","1999","10","1"],
["MDM ","Corinne","DUBOIS","F","W","620 ","Yes,for my family","71852991","2017-H1","CO ","Stevens","10","38 - 208","24 - 40","099200","S2243858A","1974","2","1"]
]`
This is the python code that I have made so fare :
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import TimeoutException
from bs4 import BeautifulSoup
from selenium.webdriver import ActionChains
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import time
import pandas as pd
import csv
driver = webdriver.Firefox()
driver.get("https://login.aviva.com.sg/directinsurance/homeinsurance.htm")
for people in profil:
dropdown_salutation =
Select(driver.find_element_by_name("person.salutationRef"))
dropdown_occupation =
Select(driver.find_element_by_name("person.occupationRef"))
dropdown_maritalstatus =
Select(driver.find_element_by_name("person.maritalstat"))
dropdown_gender=Select(driver.find_element_by_name("person.gender"))
dropdown_dobDay= Select(driver.find_element_by_name("dobDay"))
dropdown_dobMonth=Select(driver.find_element_by_name("dobMonth"))
dropdown_dobYear=Select(driver.find_element_by_name("dobYear"))
dropdown_declaration1=Select(driver.find_element_by_name("declaration1"))
dropdown_declaration2=Select(driver.find_element_by_name("declaration2"))
#now we look for all the other element that we can fill (we select by id first)
FamilyName_input = driver.find_element_by_id("surname")
GivenName_input = driver.find_element_by_id("givname")
NRIC_input = driver.find_element_by_id("nric")
PostalCode = driver.find_element_by_id("postalCode")
MobileNo = driver.find_element_by_id("textfield5")
Email = driver.find_element_by_id("email")
# Then we fill everything
dropdown_salutation.select_by_value(people[0])
GivenName_input.send_keys(people[1])
FamilyName_input.send_keys(people[2])
dropdown_gender.select_by_value(people[3])
dropdown_maritalstatus.select_by_value(people[4])
dropdown_occupation.select_by_value(people[5])
MobileNo.send_keys(people[7])
NRIC_input.send_keys(people[15])
dropdown_dobYear.select_by_value("people[16]")
dropdown_dobMonth.select_by_value(people[17])
dropdown_dobDay.select_by_value(people[18])
Email.send_keys("ada#hotmail.com")
dropdown_declaration1.select_by_value("Y")
dropdown_declaration2.select_by_value("Y")
PostalCode.send_keys(people[14])
wait = WebDriverWait(driver, 30)
# Now we can get the full address based on the postal code we provide
#here I have a first problem
driver.find_element_by_id("btnAddress").click()
wait = WebDriverWait(driver, 30)
element = wait.until(EC.element_to_be_clickable((By.ID, 'immediateFamilySaf')))
dropdown_declaration3=
Select(driver.find_element_by_name("policy.immediateFamilySaf"))
dropdown_declaration3.select_by_value("N")
# Now we click on next to move forward on the second page of the form
Next = driver.find_element_by_css_selector("a[onclick*=checkFirstTab]")
Next.click()
UnitNo =
driver.find_element_by_css_selector("a[onclick*=proceedNoUnitNo]")
UnitNo.click()
#Now we can fill the "cover needed" form
dropdown_plan=Select(driver.find_element_by_name("homeProd.planTypeRef"))
dropdown_dwelling =
Select(driver.find_element_by_name("homeProd.dwellingTypeRef"))
dropdown_insureadr=
Select(driver.find_element_by_name("homeProd.addressType"))
dropdown_coverday=Select(driver.find_element_by_name("coverStartDay"))
dropdown_covermonth=Select(driver.find_element_by_name("coverStartMonth"))
dropdown_coveryear=Select(driver.find_element_by_name("coverStartYear"))
dropdown_plan.select_by_value("HI ")
dropdown_dwelling.select_by_value(people[9])
dropdown_insureadr.select_by_value("S")
dropdown_coverday.select_by_value("1")
dropdown_covermonth.select_by_value("4")
dropdown_coveryear.select_by_value("2018")
# Now we can grab the next button and pass to the third tab
SecondTab = driver.find_element_by_name("_target0")
SecondTab.click()
#now we can grab the quote
ThirdTab = driver.find_element_by_name("_target1")
ThirdTab.click()
time.sleep(3)
driver.save_screenshot('img' + people[2] + '.png')
html= driver.page_source
doc=
# We can feed that into Beautiful Soup
doc = BeautifulSoup(html, "html.parser")
rows = doc.find('table', id='table-termsofplan').find_all('td', attrs={'class': None})
premiums = []
for row in rows:
# Find the ones that don't have 'style' as an attribute
if 'style' in row.attrs:
# Skip it! It's a header or footer row
pass
else:
premium={
'type of plan': rows[1].text,
'12 Months premium':rows[2].text,
'24 Months premium':rows[3].text,
'36 Months premium':rows[4].text,
'Total Premium 12 Months':rows[10].text,
'Total Premium 24 Months':rows[11].text,
'Total Premium 36 Months':rows[12].text,
'Goods and services Tax 12 Months':rows[14].text,
'Goods and services Tax 24 Months':rows[15].text,
'Goods and services Tax 36 Months':rows[16].text,
'Single Payment 12 Months':rows[19].text,
'Single Payment 24 Months':rows[20].text,
'Single Payment 36 Months':rows[21].text,
}
premiums.append(premium)
driver.get("https://login.aviva.com.sg/directinsurance/homeinsurance.htm")
driver.close()
import pandas as pd
premium_df = pd.DataFrame(premiums)
premium_df.to_csv("premium.csv", index=False)