I am trying to scrape an airbnb listing. I cant figure out a way to get the full list of amenities other than clicking on "more". I am using selenium to simulate the click, but it does nt seem to work.
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
url = 'https://www.airbnb.com/rooms/4660676'
driver = webdriver.Firefox()
driver.get(url)
elem = driver.find_element_by_xpath('//a[#class="expandable-trigger-more"]')
actions.click(elem).perform()
The XPath itself is correct, but you have not defined the actions:
from selenium.webdriver.common.action_chains import ActionChains
elem = driver.find_element_by_xpath('//a[#class="expandable-trigger-more"]')
actions = ActionChains(driver)
actions.click(elem).perform()
Note that you can simply use the WebElement's click() method instead:
elem = driver.find_element_by_xpath('//a[#class="expandable-trigger-more"]')
elem.click()
Works for me.
If you are getting NoSuchElementException error, you might need to wait for the link to be clickable via WebDriverWait and element_to_be_clickable.
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
element = WebDriverWait(driver, 10).until(
EC.element_to_be_clickable((By.XPATH, '//a[#class="expandable-trigger-more"]'))
)
element.click()
A very simple way of achieving this is given below
driver.find_element_by_xpath('//a[#class="expandable-trigger-more"]').click()
It works for me hope will work for you as well.
Related
I'm using selenium 4 with python to automate webpage.
Click() method is not working.
Able to click a button using submit() method only if the path has type=submit, but couldnt able to select a link using click method. Please help if anyone knows alternative option for click() method to click a link.
The following is the code:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
s = Service("C:/Program Files/webdriver/Chrome Driver/chromedriver.exe")
driver = webdriver.Chrome(service=s)
driver.get("https://www.sololearn.com/users/login")
time.sleep(2)
#driver.find_element(By.CLASS_NAME, "sl-login-login-form").send_keys(Keys.ENTER)
#---click here fails--------
driver.find_element(By.CLASS_NAME, "sl-login-login-form").click()
#---click here fails
#driver.find_element(By.CLASS_NAME, "sl-login-login-form").submit()
#login.click()
time.sleep(5)
driver.quit()
**Also i have tried with the below code but still it is not working:**
driver.find_element(By.CLASS_NAME, "sl-login-login-form").send_keys(Keys.ENTER)
driver.find_element(By.CLASS_NAME, "sl-login-login-form").submit()
login.click()
Try the below
element = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CLASS_NAME, "CLASSNAME")))
element.click();
OR
element = driver.find_element(By.CLASS_NAME, "className")
driver.execute_script("arguments[0].click();", element)
Do not forget to import
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import org.openqa.selenium.JavascriptExecutor
I am trying to get the review of a certain product but it returns an error.
My code:
import selenium
from selenium import webdriver
chrome_path = r"C:\Users\AV\AppData\Local\Programs\Python\Python39\Scripts\chromedriver.exe"
driver = webdriver.Chrome(chrome_path)
driver.get("https://oldnavy.gapcanada.ca/browse/product.do?pid=647076053&cid=1180630&pcid=26190&vid=1&nav=meganav%3AWomen%3ADeals%3ASale&grid=pds_0_1034_1#pdp-page-content")
driver.execute_script("window.scrollTo(0, 1000)")
import time
from time import sleep
sleep(5)
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div.promoDrawer__handlebar__icon"))).click()
review = driver.find_elements_by_class_name("pr-rd-description-text")
for post in review:
print(post.text)
driver.find_element_by_xpath('//*[#id="pr-review-display"]/footer/div/div/a').click()
review2 = driver.find_elements_by_class_name("pr-rd-description-text")
for post in review2:
print(post.text)
It returns: selenium.common.exceptions.ElementNotInteractableException: Message: element not interactable
Can you please tell me what should I do?
The button is really hard to click.
I guess it could be achieved by adding some more waits and moving with ActionChains class methods.
I could click it with Javascript code with no problems.
What it does:
1 Scrolls to the Next button
2 Clicks it.
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
driver = webdriver.Chrome(executable_path='/snap/bin/chromium.chromedriver')
driver.get(
"https://oldnavy.gapcanada.ca/browse/product.do?pid=647076053&cid=1180630&pcid=26190&vid=1&nav=meganav%3AWomen%3ADeals%3ASale&grid=pds_0_1034_1#pdp-page-content")
driver.execute_script("window.scrollTo(0, 1000)")
wait = WebDriverWait(driver, 20)
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div.promoDrawer__handlebar__icon"))).click()
wait.until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, ".pr-rd-description-text")))
review = driver.find_elements_by_css_selector(".pr-rd-description-text")
for post in review:
print(post.text)
# wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, ".pr-rd-pagination-btn"))).click()
element = driver.find_element_by_css_selector(".pr-rd-pagination-btn")
# actions = ActionChains(driver)
# actions.move_to_element(element).click().perform()
driver.execute_script("arguments[0].scrollIntoView();", element) # Scrolls to the button
driver.execute_script("arguments[0].click();", element) # Clicks it
print("clicked next")
I also rearranged your code, moved imports to the beginning of the file, got rid of unpredictable time.sleep() and used more reliable css locators. However, your locator should also work.
I left the options I tried commented out.
That element is weird. Even when I scroll into view, use actions to click it, or execute javascript to click, it doesn't work. What I would suggest is just grabbing the href attribute from the element and going to that URL, using something like this:
driver.get(driver.find_element_by_xpath('//*[#id="pr-review-display"]/footer/div/div/a').get_attribute('href'))
I want to click on button on web site, but this button appears after you put your cursor on the right place, otherwise you can not see this button. I tried to ignore that and do just normal click by finding the element by xpath. But that did not work out, there was mistake:
selenium.common.exceptions.ElementNotInteractableException: Message: element not
interactable: element has zero size
How to deal with that?
To hover over the element you can use something like this:
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
chrome= webdriver.Chrome()
chrome.get('http://foo.bar')
element = chrome.find_element_by_css_selector("the_element_u_want")
hover = ActionChains(chrome).move_to_element(element)
hover.perform()
if the element takes time to appear use selenium's wait or time.sleep()
find more info about ActionChains here
this is some code I found here.
Usually, the ElementNotInteractableException is thrown when the element hasn't loaded completely. Here's a possible solution:
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as ec
driver = webdriver.Chrome('C:\\Users\\dell\\Libs\\chromedriver.exe')
driver.get("http://url")
driver.maximize_window()
# wait for element to appear, then hover it
wait = WebDriverWait(driver, 10)
men_menu = wait.until(ec.visibility_of_element_located((By.XPATH, "//a[#data-tracking-id='men']")))
ActionChains(driver).move_to_element(men_menu).perform()
This method worked well for me.
def hoverXPATH(xpath):
item = WebDriverWait(driver, 10)\
.until(EC.visibility_of_element_located((By.XPATH, xpath)))
ActionChains(driver).move_to_element(item).perform()
I am pretty new in selenium and I was looking forward to execute a click() button that expands a table, and after that I was looking forward to scrape the content of the expanded table.
This is what I tried:
url = 'https://www.telegraph.co.uk/markets-hub/assets/shares/'
phantomjs_path = '/usr/local/bin/phantomjs'
driver = webdriver.PhantomJS(executable_path=phantomjs_path,service_args=['--load-images=no'])
driver.get(url)
element = driver.find_element_by_xpath("//div[#class='tmg-show-more']")
html_source = driver.page_source
element.click()
driver.quit()
soup = BeautifulSoup(html_source,"html5lib")
tables=soup.find('table',class_="table-static kurser-table")
link=pd.read_html(str(tables),flavor='html5lib',thousands='.',decimal=',',header=0)
print(link)
The current output is as it follows:
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.ElementNotVisibleException: Message: {"errorMessage":"Element is not currently visible and may not be manipulated",
"request":{"headers":{"Accept":"application/json","Accept-Encoding":"identity","Connection":"close","Content-Length":"81","Content-
Type":"application/json;charset=UTF-8","Host":"127.0.0.1:58447","User-Agent":"Python http auth"},
"httpVersion":"1.1","method":"POST","post":"{\"id\": \":wdc:1556893378248\", \"sessionId\": \"f223cb80-6dae-11e9-b00b-4bb158952171\"}",
"url":"/click","urlParsed":{"anchor":"","query":"","file":"click","directory":"/","path":"/click","relative":"/click","port":"","host":"","password":"","user":"",
"userInfo":"","authority":"","protocol":"","source":"/click","queryKey":{},"chunks":["click"]},"urlOriginal":"/session/f223cb80-6dae-11e9-b00b-4bb158952171/element/:wdc:1556893378248/click"}}
Screenshot: available via screen
You would need to scroll down to let the selenium web driver knows where the element is actually present.
Code :
driver.get("https://www.telegraph.co.uk/markets-hub/assets/shares/")
wait = WebDriverWait(driver, 10)
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, 'div.section-box.kurser-table-container')))
element = driver.find_element_by_xpath("//a[#ng-click='stocksShowMore()']")
actions = ActionChains(driver)
actions.move_to_element(element).perform()
element.click()
Imports :
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
You are getting ElementNotVisibleException means you are trying to click the element before it is visible.
Also, you are getting the page source before clicking the button. You need to get the source after clicking and loading the new data.
Try the following code before getting the page source.
wait = WebDriverWait(driver, 20)
element = wait.until(EC.element_to_be_clickable(driver.find_element_by_xpath("//div[#class='tmg-show-more']")))
element.click()
wait.until(EC.invisibility_of_element(element))
To use the WebDriverWait you need to import these
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
I am trying to post something on http://indianrailforums.in using selenium script. I am able to login and reach this page: http://indiarailinfo.com/blog using the selenium script, but after I click post button I am not able to send text in the text area.
This is my code:
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Firefox()
driver.get("http://indiarailinfo.com")
element = driver.find_element_by_name("e")
element.send_keys("yagyank#gmail.com")
element = driver.find_element_by_name("p")
element.send_keys("suvrit")
element.submit()
driver.get("http://indiarailinfo.com/blog")
element = driver.find_element_by_link_text('Post')
element.click()
element = driver.find_element_by_xpath("/html/body/div[1]/div[5]/table/tbody/tr/td[1]/div[1]/div[1]/div/div/form/textarea")
#element.sendKeys(Keys.HOME + "abc");
#element = driver.find_element_by_name("TrainBlog")
element.send_keys("suvrit")
#driver.quit()
EDIT: Issue solved by using submit button and using driver.implicitly_wait(10) just before calling xpath
I was able to post with PhantomJS driver. It should work with Firefox either.
My code is:
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.PhantomJS()
driver.get("http://indiarailinfo.com")
element = driver.find_element_by_name("e")
element.send_keys("yagyank#gmail.com")
element = driver.find_element_by_name("p")
element.send_keys("suvrit")
element.submit()
driver.get("http://indiarailinfo.com/blog")
element = driver.find_element_by_link_text('Post')
element.click()
element = driver.find_element_by_xpath("/html/body/div[1]/div[5]/table/tbody/tr/td[1]/div[1]/div[1]/div/div/form/textarea")
print element.tag_name
#element.sendKeys(Keys.HOME + "abc");
#element = driver.find_element_by_name("TrainBlog")
element.send_keys("chuh-pook")
element.submit()
#driver.quit()
The problem is that you have not submitted the form. Either call submit():
element = driver.find_element_by_xpath("/html/body/div[1]/div[5]/table/tbody/tr/td[1]/div[1]/div[1]/div/div/form/textarea")
element.send_keys("suvrit")
element.submit() # it would find the parent form and submit it
Or, click the post button:
post = driver.find_element_by_css_selector('input.postbtn')
post.click()