I have written a code than logins to a webpage and tries to download a file. but after login and before downloading it requires to select a particular field from drop down and then select a particular tab to download the file, can anyone pls assist what could be done here ?
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(service=webdriver_service, options=options)
wait = WebDriverWait(driver, 20)
url = "products.markit.com/home/login.jsp"
driver.get(url)
wait.until(EC.element_to_be_clickable((By.NAME, "username"))).send_keys("Admin")
wait.until(EC.element_to_be_clickable((By.NAME, "password"))).send_keys("admin123")
wait.until(EC.element_to_be_clickable((By.TAG_NAME, "button"))).click()
Inspect element for first selection (Pricing Data - Loan)
Inspect element for Past Loan Market
Inspect element for Download tab
Related
I need to scrape this site. How can I choose a certain element from the 'Licensee Name' dropdown list?
I tried the following code, but it's not working (I tried using the class attribute, but it's not working either):
input_ = driver.find_element(By.XPATH, "//input[#class='select2-search__field']")
input_.send_keys("1x2 NETWORK MALTA LTD")
After clicking on the drop-down, you need to insert the input text into the input field, then click on appeared search result and finally click on Search button.
The following code works:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(options=options, service=webdriver_service)
wait = WebDriverWait(driver, 10)
url = "https://www.mga.org.mt/licences/"
driver.get(url)
wait.until(EC.frame_to_be_available_and_switch_to_it((By.XPATH, "//iframe[contains(#src,'//mgali')]")))
wait.until(EC.element_to_be_clickable((By.XPATH, "//span[#id='select2-ListBox3-container']"))).click()
wait.until(EC.element_to_be_clickable((By.CLASS_NAME, "select2-search__field"))).send_keys("1x2 NETWORK MALTA LTD")
wait.until(EC.element_to_be_clickable((By.XPATH, "//li[contains(#class,'select2-results__option')][contains(.,'NETWORK MALTA')]"))).click()
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, '[type="submit"]'))).click()
The result is:
I am trying to scrap articles from this website. I manage to do the login part but when I try to click on the search button and send the values I got a timeout error. I try running the selenium with start-maximize option and I noticed the page don't seem to load.
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, '//*[#id="search__input"]')))
input_text = elem.find_element(by=By.XPATH, value='//*[#id="search__input"]').click()
input_text.send_keys("Anthony Albanese")
print(input_text.get_attribute('value'))
I have tried to use the get_attribute('innerHtml') and I got the search button HTML but I have to send the keys so it does not seems to be of any use. This is the error that I got
screenshot. What should I do to send in the search terms?
There are 2 elements on that page matching //*[#id="search__input"] XPath locator, while you need the second one.
You have to use unique locator.
This should work better:
text_input = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//div[#class='header__search']//*[#id='search__input']")))
text_input.click()
text_input.send_keys("Anthony Albanese")
This can also be done with CSS Selectors. They are shorter in this case:
text_input = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, ".header__search #search__input")))
text_input.click()
text_input.send_keys("Anthony Albanese")
UPD
This is the code I used, exactly accordingly to what I wrote before:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(options=options, service=webdriver_service)
wait = WebDriverWait(driver, 20)
url = "https://www.theaustralian.com.au/"
driver.get(url)
text_input = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, ".header__search #search__input")))
text_input.click()
text_input.send_keys("Anthony Albanese")
The web page after the code above looks as following:
And if you add an Enter click to the search input as following:
from selenium import webdriver
from selenium.webdriver import Keys
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(options=options, service=webdriver_service)
wait = WebDriverWait(driver, 20)
url = "https://www.theaustralian.com.au/"
driver.get(url)
text_input = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, ".header__search #search__input")))
text_input.click()
text_input.send_keys("Anthony Albanese" + Keys.ENTER)
the web page will finally become as following
I'm using the https://www.perlego.com/ website to do some scraping for books, my current problem is that i'm trying to search using the isbn Number of the book so the webdriver gets this following page for example https://www.perlego.com/search?query=9780717183241
However, using page inspection on the website, i cannot find the element that i can click on using webdriver
the idea is to click on the book and load the page that comes with it
The following code closes the cookies banner and clicks on the book link and opens it
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(options=options, service=webdriver_service)
wait = WebDriverWait(driver, 10)
url = "https://www.perlego.com/search?query=9780717183241&tab=book&page=1&language=All%20languages&publicationDate=&topic=&publisher=&author=&format="
driver.get(url)
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button[data-testid*='Cookies']"))).click()
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "a[data-test-locator*='new-book']"))).click()
I am trying to scraping in this URL, dealing with a Download button and I am having a problem, as the last line gives a ElementClickInterceptedException.
My actual goal is to download the CSV file.
The code:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.chrome.options import Options
from ipykernel import kernelapp as app
import time
options = webdriver.ChromeOptions()
driver_path = 'C:\\Users\\Idener\\Downloads\\chromedriver_win32\\chromedriver.exe'
driver = webdriver.Chrome(driver_path, options=options)
url = "https://pubchem.ncbi.nlm.nih.gov/compound/2078"
driver.get(url)
wait = WebDriverWait(driver, 5)
wait.until(EC.element_to_be_clickable((By.XPATH, '//*[#id="NIOSH-Toxicity-Data"]/div[1]/div/div/a'))).click()
wait = WebDriverWait(driver, 10)
wait.until(EC.element_to_be_clickable((By.XPATH, '//*[#id="Download"]'))).click()
enter image description here
Element you trying to click in initially out of the visible viewpoint. So, you need first to scroll the page and only then to click on that element.
By clicking the first element new tab is opened and the second element you want to click is there, on the second tab. So, you need to switch to the new tab to access that element.
No need to define wait = WebDriverWait(driver, 10) second time.
The following code is working:
import time
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(options=options, service=webdriver_service)
wait = WebDriverWait(driver, 10)
url = "https://pubchem.ncbi.nlm.nih.gov/compound/2078#section=Toxicity"
driver.get(url)
element = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "#NIOSH-Toxicity-Data a[title*='Open']")))
element.location_once_scrolled_into_view
time.sleep(1)
element.click()
driver.switch_to.window(driver.window_handles[1])
wait.until(EC.element_to_be_clickable((By.ID, "Download"))).click()
It does not download the file, only opens the downloading dialog
zillow picture
I have the above image at https://www.zillow.com/homes/for_sale/7132668_zpid/globalrelevanceex_sort/60.780619,-65.522461,4.521666,-125.551758_rect/3_zm/
I cant seem to find the selector for tax history.
I tried to use driver wait but the table that is output is the price history not tax history.
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import NoSuchElementException
options = webdriver.ChromeOptions()
options.add_argument(f"user-agent={useragent[0]}")
options.add_argument('--proxy-server=%s' % ips[0])
options.add_argument('--incognito')
chromedriver = '~/Downloads/chromedriver'
chromedriver = os.path.expanduser(chromedriver)
driver = webdriver.Chrome(chromedriver, chrome_options=options)
driver.get('https://www.zillow.com/homes/for_sale/7132668_zpid/globalrelevanceex_sort/60.673178,-74.663086,4.653079,-116.323243_rect/3_zm/2_p/')
wait(driver, 10).until(EC.element_to_be_clickable((By.CLASS_NAME, "hdp-collapse"))).click()
table = wait(driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, "div#hdp-tax-history")))
Looks like you need some waits (and maybe some clicks to get that tab visible. You can write out the table. The below is just to show how you can access
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
url = 'https://www.zillow.com/homes/for_sale/7132668_zpid/globalrelevanceex_sort/53.566414,-73.081055,17.434511,-118.081055_rect/3_zm/'
d = webdriver.Chrome()
d.get(url)
WebDriverWait(d,20).until(EC.presence_of_element_located((By.ID , 'price-and-tax-history'))).click()
tabs = WebDriverWait(d,5).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".zsg-tab-link")))
tabs[1].click()
print(d.find_element_by_css_selector('#hdp-tax-history table').text) # just to show present