Selenium seems to be looking for data in an old page and not the new one.
I'm trying to automate a search where I select from a dropdown menu and fill a box with some value
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select
url = 'http://www.op.nysed.gov/opsearches.htm'
value = '60'
license_no = '084157'
driver = webdriver.Chrome()
driver.get(url)
select = Select(driver.find_element(By.XPATH, '//form[#id="licensee-num-form"]/center/select'))
select.select_by_value(value)
fill = driver.find_element(By.XPATH, '//form[#id="licensee-num-form"]/center/input')
fill.send_keys(license_no)
fill.send_keys(Keys.ENTER)
data = driver.find_element(By.XPATH, "//div[#id='content_column']")
However, when I print data.text, it prints data from the first page, not the second one. I tried using driver.refresh() to refresh the page but it did not work.
This happens because you getting the data = driver.find_element(By.XPATH, "//div[#id='content_column']") immediately after clicking the Enter. The page still not refreshed. You should add a short delay there and then wait for element visibility on the refreshed page.
Please try this:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import Select
url = 'http://www.op.nysed.gov/opsearches.htm'
value = '60'
license_no = '084157'
driver = webdriver.Chrome()
wait = WebDriverWait(driver, 20)
driver.get(url)
select = Select(driver.find_element(By.XPATH, '//form[#id="licensee-num-form"]/center/select'))
select.select_by_value(value)
fill = driver.find_element(By.XPATH, '//form[#id="licensee-num-form"]/center/input')
fill.send_keys(license_no)
fill.send_keys(Keys.ENTER)
time.sleep(0.5)
data = wait.until(EC.visibility_of_element_located((By.XPATH, "//div[#id='content_column']")))
Related
I would like to extract the text that appears on mouse hover on an element from the website https://idsc.cidadessustentaveis.org.br/rankings. A particular example of text of interest, is the text “Erradicacao da pobreza Pontuacao: 44,47” which appears on hovering the first bar located in the column “Desempenho por ODS”. I have tried the code below but it returns the blank text ‘’
from selenium import webdriver
driver = webdriver.Firefox()
driver.get("https://idsc.cidadessustentaveis.org.br/rankings")
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
wait = WebDriverWait(driver, 20)
desired_elem = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, '.SdgPerformanceBar__Block-sc-1yl1q71-2.fBQLcJ')))
I printed an attribute of the element extracted which confirmed that I have successfully gotten to the targeted element.
print(desired_elem.get_attribute('outerHTML'))
Which returned:
<div style="width:2.62%" class="SdgPerformanceBar__Block-sc-1yl1q71-2 fBQLcJ"></div>
Note that by inspecting the element in Firefox I found that the element has no innerHTML.
I then tried to extract the text using desired_elem.text but I get a blank ‘’
I also tried the code below which returned a blank as well.
from selenium.webdriver.common.action_chains import ActionChains
elem = driver.find_element(By.CSS_SELECTOR, '.SdgPerformanceBar__Block-sc-1yl1q71-2.fBQLcJ');
actions = ActionChains(driver)
actions.move_to_element(elem)
actions.move_to_element(elem).perform()
Calling elem.text returned ''
You was close to the solution.
When hovering over those elements tooltips appearing.
These tooltips will present different texts according to element you hovered over.
Here I used your code, just added the tooltips
from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(options=options, service=webdriver_service)
wait = WebDriverWait(driver, 10)
actions = ActionChains(driver)
tooltip1 = "div[role='tooltip'] .MuiTypography-root.MuiTypography-body1"
tooltip2 = "div[role='tooltip'] .MuiTypography-root.MuiTypography-body2"
url = "https://idsc.cidadessustentaveis.org.br/rankings"
driver.get(url)
desired_elem = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, '.SdgPerformanceBar__Block-sc-1yl1q71-2.fBQLcJ')))
actions.move_to_element(desired_elem).perform()
tt1_text = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, tooltip1))).text
tt2_text = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, tooltip2))).text
print(tt1_text)
print(tt2_text)
The output is:
Erradicação da pobreza
Pontuação: 44,47
zillow picture
I have the above image at https://www.zillow.com/homes/for_sale/7132668_zpid/globalrelevanceex_sort/60.780619,-65.522461,4.521666,-125.551758_rect/3_zm/
I cant seem to find the selector for tax history.
I tried to use driver wait but the table that is output is the price history not tax history.
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import NoSuchElementException
options = webdriver.ChromeOptions()
options.add_argument(f"user-agent={useragent[0]}")
options.add_argument('--proxy-server=%s' % ips[0])
options.add_argument('--incognito')
chromedriver = '~/Downloads/chromedriver'
chromedriver = os.path.expanduser(chromedriver)
driver = webdriver.Chrome(chromedriver, chrome_options=options)
driver.get('https://www.zillow.com/homes/for_sale/7132668_zpid/globalrelevanceex_sort/60.673178,-74.663086,4.653079,-116.323243_rect/3_zm/2_p/')
wait(driver, 10).until(EC.element_to_be_clickable((By.CLASS_NAME, "hdp-collapse"))).click()
table = wait(driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, "div#hdp-tax-history")))
Looks like you need some waits (and maybe some clicks to get that tab visible. You can write out the table. The below is just to show how you can access
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
url = 'https://www.zillow.com/homes/for_sale/7132668_zpid/globalrelevanceex_sort/53.566414,-73.081055,17.434511,-118.081055_rect/3_zm/'
d = webdriver.Chrome()
d.get(url)
WebDriverWait(d,20).until(EC.presence_of_element_located((By.ID , 'price-and-tax-history'))).click()
tabs = WebDriverWait(d,5).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".zsg-tab-link")))
tabs[1].click()
print(d.find_element_by_css_selector('#hdp-tax-history table').text) # just to show present
I'm Working on a data scraping work by using python and I wanted to do scrape the new redirect page data after clicking on the redirect button.
This is the code which i have tried.
browser = webdriver.Firefox()
browser.get("https://www.cbsl.gov.lk/en/statistics/economic-indicators")
window_before = browser.window_handles[0]
print(window_before)
browser.find_element_by_xpath('/html/body/div[2]/div[3]/div/div/div/div/div/div/div/div/div/div/div/div/div/div/div/div[4]/div[2]/p[1]/a').click()
window_after = browser.window_handles[1]
browser.switch_to_window(window_after)
print(window_after)
bs_obj = BSoup(browser.page_source,'lxml')
table = bs_obj.find("table", id="statTB")
print(table)
this will redirect to the new page. but after print the table it was not showing anything. I think still it was trying on the old page.
No. When you switched to new window, browser.page_source returns you HTML of new
window, but you might need to wait until required table appeared in DOM:
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
...
browser.switch_to_window(window_after)
table = WebDriverWait(browser, 10).until(EC.visibility_of_element_located((By.ID, "statTB")))
print(table.text)
you need multiple WebDriverWait, waiting second window and page loaded
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
browser.get("https://www.cbsl.gov.lk/en/statistics/economic-indicators")
window_before = browser.window_handles[0]
print(window_before)
browser.find_element_by_xpath('/html/body/div[2]/div[3]/div/div/div/div/div/div/div/div/div/div/div/div/div/div/div/div[4]/div[2]/p[1]/a').click()
WebDriverWait(browser, 20).until(EC.number_of_windows_to_be(2))
window_after = browser.window_handles[1]
browser.switch_to_window(window_after)
print(window_after)
myElem = WebDriverWait(browser, 10).until(EC.presence_of_element_located((By.ID, 'Grid')))
bs_obj = BeautifulSoup(browser.page_source, 'lxml')
table = bs_obj.find("table", id="statTB")
print(table)
I am trying to write a script that uses selenium to download many files which consist of different NHL players information; game-log. I want to download a file for each players in the following table: https://www.naturalstattrick.com/playerteams.php?fromseason=20142015&thruseason=20162017&stype=2&sit=all&score=all&stdoi=std&rate=y&team=ALL&pos=S&loc=B&toi=0.1&gpfilt=none&fd=&td=&tgp=410&lines=single
Once on that website, I wanted to click on all the players' name in the table. When a player's name is clicked through the href link, a new window opens. There are few drop-down menus at the top. I want to select "Rate" instead of "Counts" and also select " Game Log" instead of "Player Summary", and then click "Submit". Finally, I want to click on CSV(All) at the bottom to download a CSV file.
Here is my current code:
from selenium import webdriver
import csv
from selenium.webdriver.support.ui import Select
from datetime import date, timedelta
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
chromedriver =("C:/Users/Michel/Desktop/python/package/chromedriver_win32/chromedriver.exe")
driver = webdriver.Chrome(chromedriver)
driver.get("https://www.naturalstattrick.com/playerteams.php?fromseason=20142015&thruseason=20162017&stype=2&sit=all&score=all&stdoi=std&rate=y&team=ALL&pos=S&loc=B&toi=0.1&gpfilt=none&fd=&td=&tgp=410&lines=single")
table = driver.find_element_by_xpath("//table[#class='indreg dataTable no-footer DTFC_Cloned']")
for row in table.find_elements_by_xpath("//tr[#role='row']")
links = driver.find_element_by_xpath('//a[#href]')
links.click()
select = Select(driver.find_element_by_name('rate'))
select.select_by_value("y")
select1 = Select(driver.find_element_by_name('v'))
select1.select_by_value("g")
select2 = Select(driver.find_element_by_type('submit'))
select2.select_by_value("submit")
WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH , '//div[#class="dt-button button-csv button-htm15"]')))
CSVall = driver.find_element_by_xpath('//div[#class="dt-button button-csv button-htm15"]')
CSVall.click()
driver.close()
I try to change different things, but I always get an error. Where is the problem ?
Moreover, I think I should probably add a line to wait for the website to load because it takes a few seconds; after "driver.get". I do not know what should be the expected conditions to end the wait in this case.
Thanks
Rather than keep clicking through selections you could grab the playerIds from the first page and concantenate those, along with the strings representing the selections for Rate and Game Log into the queryString part of the new URL. Sure you can tidy up the following.
from selenium import webdriver
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
def getPlayerId(url):
id = url.split('playerid=')[1]
id = id.split('&')[0]
return id
def makeNewURL(playerId):
return 'https://www.naturalstattrick.com/playerreport.php?fromseason=20142015&thruseason=20162017&stype=2&sit=all&stdoi=oi&rate=y&v=g&playerid=' + playerId
#chromedriver =("C:/Users/Michel/Desktop/python/package/chromedriver_win32/chromedriver.exe")
driver = webdriver.Chrome()
driver.get("https://www.naturalstattrick.com/playerteams.php?fromseason=20142015&thruseason=20162017&stype=2&sit=all&score=all&stdoi=std&rate=y&team=ALL&pos=S&loc=B&toi=0.1&gpfilt=none&fd=&td=&tgp=410&lines=single")
links = driver.find_elements_by_css_selector('table.indreg.dataTable.no-footer.DTFC_Cloned [href*=playerid]')
newLinks = []
for link in links:
newLinks.append(link.get_attribute('href'))
for link in newLinks:
playerId = getPlayerId(link)
link = makeNewURL(playerId)
driver.get(link)
WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH , '//a[#class="dt-button buttons-csv buttons-html5"][2]')))
CSVall = driver.find_element_by_xpath('//a[#class="dt-button buttons-csv buttons-html5"][2]')
CSVall.click()
you don't need to click each player link but save the URLs as list, also there are several error, you can see working code below
from selenium import webdriver
import csv
from selenium.webdriver.support.ui import Select
from datetime import date, timedelta
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
chromedriver =("C:/Users/Michel/Desktop/python/package/chromedriver_win32/chromedriver.exe")
driver = webdriver.Chrome(chromedriver)
driver.get("https://www.naturalstattrick.com/playerteams.php?fromseason=20142015&thruseason=20162017&stype=2&sit=all&score=all&stdoi=std&rate=y&team=ALL&pos=S&loc=B&toi=0.1&gpfilt=none&fd=&td=&tgp=410&lines=single")
playerLinks = driver.find_elements_by_xpath("//table[#class='indreg dataTable no-footer DTFC_Cloned']//a")
playerLinks = [p.get_attribute('href') for p in playerLinks]
print(len(playerLinks))
for url in playerLinks:
driver.get(url)
select = Select(driver.find_element_by_name('rate'))
select.select_by_value("y")
select1 = Select(driver.find_element_by_name('v'))
select1.select_by_value("g")
driver.find_element_by_css_selector('input[type="submit"]').click()
WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH , '//a[#class="dt-button buttons-csv buttons-html5"][2]')))
CSVall = driver.find_element_by_xpath('//a[#class="dt-button buttons-csv buttons-html5"][2]')
CSVall.click()
driver.close()
I want to select 1st auto suggested value after fill the data but it is not working for me
WebDriverWait(driver, 10).until(driver.find_element_by_xpath("//*[#id='txtFrom']").send_keys("Delhi, India",Keys.DOWN))
I tried using both way Keys.TAB and Keys.DOWN but when I am using this that time even data is not getting filed in textbox and error is coming
WebDriverWait(driver, 10).until(driver.find_element_by_xpath("//*[#id='txtFrom']").send_keys("Delhi, India",Keys.DOWN)) NameError: name 'Keys' is not defined
Here is my Complete Code
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
path_to_chromedriver = 'C:/Users/prash/Desktop/WebScrape/selenium/chromedriver'
driver = webdriver.Chrome(executable_path = path_to_chromedriver)
driver.implicitly_wait(10)
driver.maximize_window()
url = 'http://tis.nhai.gov.in/tollplazasonmap?language=en'
driver.get(url)
embed = driver.find_element_by_tag_name('embed')
driver.switch_to.frame(embed)
element = driver.find_element_by_id('tollstation')
driver.execute_script("arguments[0].click();", element)
WebDriverWait(driver, 10).until(driver.find_element_by_xpath("//*[#id='txtFrom']").send_keys("Delhi, India",Keys.DOWN))
WebDriverWait(driver, 10).until(driver.find_element_by_xpath("//*[#id='txtTo']").send_keys("Bangalore",Keys.DOWN))
element2 = driver.find_element_by_xpath("//*[#id='showstation']/p[7]/a")
driver.execute_script("arguments[0].click();", element2)
To able to use Keys module you should import it first as
from selenium.webdriver.common.keys import Keys
After that you can use in your code Keys.TAB, Keys.DOWN, etc