I am trying to get the review of a certain product but it returns an error.
My code:
import selenium
from selenium import webdriver
chrome_path = r"C:\Users\AV\AppData\Local\Programs\Python\Python39\Scripts\chromedriver.exe"
driver = webdriver.Chrome(chrome_path)
driver.get("https://oldnavy.gapcanada.ca/browse/product.do?pid=647076053&cid=1180630&pcid=26190&vid=1&nav=meganav%3AWomen%3ADeals%3ASale&grid=pds_0_1034_1#pdp-page-content")
driver.execute_script("window.scrollTo(0, 1000)")
import time
from time import sleep
sleep(5)
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div.promoDrawer__handlebar__icon"))).click()
review = driver.find_elements_by_class_name("pr-rd-description-text")
for post in review:
print(post.text)
driver.find_element_by_xpath('//*[#id="pr-review-display"]/footer/div/div/a').click()
review2 = driver.find_elements_by_class_name("pr-rd-description-text")
for post in review2:
print(post.text)
It returns: selenium.common.exceptions.ElementNotInteractableException: Message: element not interactable
Can you please tell me what should I do?
The button is really hard to click.
I guess it could be achieved by adding some more waits and moving with ActionChains class methods.
I could click it with Javascript code with no problems.
What it does:
1 Scrolls to the Next button
2 Clicks it.
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
driver = webdriver.Chrome(executable_path='/snap/bin/chromium.chromedriver')
driver.get(
"https://oldnavy.gapcanada.ca/browse/product.do?pid=647076053&cid=1180630&pcid=26190&vid=1&nav=meganav%3AWomen%3ADeals%3ASale&grid=pds_0_1034_1#pdp-page-content")
driver.execute_script("window.scrollTo(0, 1000)")
wait = WebDriverWait(driver, 20)
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div.promoDrawer__handlebar__icon"))).click()
wait.until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, ".pr-rd-description-text")))
review = driver.find_elements_by_css_selector(".pr-rd-description-text")
for post in review:
print(post.text)
# wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, ".pr-rd-pagination-btn"))).click()
element = driver.find_element_by_css_selector(".pr-rd-pagination-btn")
# actions = ActionChains(driver)
# actions.move_to_element(element).click().perform()
driver.execute_script("arguments[0].scrollIntoView();", element) # Scrolls to the button
driver.execute_script("arguments[0].click();", element) # Clicks it
print("clicked next")
I also rearranged your code, moved imports to the beginning of the file, got rid of unpredictable time.sleep() and used more reliable css locators. However, your locator should also work.
I left the options I tried commented out.
That element is weird. Even when I scroll into view, use actions to click it, or execute javascript to click, it doesn't work. What I would suggest is just grabbing the href attribute from the element and going to that URL, using something like this:
driver.get(driver.find_element_by_xpath('//*[#id="pr-review-display"]/footer/div/div/a').get_attribute('href'))
Related
I am trying to scrape some google data but I first want to click the 'I agree' button that google pops up. This is the script I use to do that:
import time
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
search_question = input("Ask a question: ")
driver = webdriver.Chrome("*Your Webdriver location*")
driver.wait = WebDriverWait(driver, 5)
driver.get("https://google.com")
time.sleep(1)
agree = driver.wait.until(EC.presence_of_element_located((By.XPATH, '//*[#id="introAgreeButton"]/span/span')))
agree.click()
# time.sleep(0.2)
search = driver.find_element_by_class_name("gLFyf")
search.send_keys(search_question)
search.send_keys(Keys.ENTER)
The problem is selenium doesn't seem to locate the button and therefore I get a timeout error. (I have tried also with find_element_by_xpath and still not working).
If you scroll up in the devtools inspector you'll notice that your element is within an iframe:
You need to switch to that frame first, click your button then switch back to the default content (the main page)
driver.get("https://google.com")
#active the iframe and click the agree button
WebDriverWait(driver, 10).until(EC.frame_to_be_available_and_switch_to_it((By.XPATH, "//iframe")))
agree = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//*[#id="introAgreeButton"]/span/span')))
agree.click()
#back to the main page
driver.switch_to_default_content()
That works for me.
FYI - There's only 1 iframe on the page, that's why the xpath //iframe works. If there were multiple you'd need to identify it with higher accuracy.
If you already have been agreed,then agree button would not appear. That why it is not be able to find given XPath.
Try this:
import time
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
search_question = input("Ask a question: ")
driver = webdriver.Chrome(".\chromedriver.exe")
driver.wait = WebDriverWait(driver, 5)
driver.get("https://google.com")
time.sleep(3)
# agree = driver.wait.until(EC.presence_of_element_located((By.XPATH, '//*[#id="introAgreeButton"]/span/span')))
# agree.click()
# time.sleep(0.2)
search = driver.find_element_by_class_name("gLFyf")
search.send_keys(search_question)
search.send_keys(Keys.ENTER)
Managed to get it working.
Seems like one of the few elements that are interactive in the popup when you first load the site is the language dropdown, which I found by class name.
Then the Agree button is detectable and you can find it by class too.
driver.get("https://www.google.com")
driver.maximize_window()
time.sleep(2)
#finds+clicks language dropdown, interaction unhides the rest of the popup html
driver.find_element_by_class_name('tHlp8d').click()
time.sleep(2)
#finds+clicks the agree button now that it has become visible
driver.find_element_by_id('L2AGLb').click()
you should now have the standard searchbar etc
i had some problems with clicking pop-ups and the problem turned out to be with that click().
im not sure that it might be the same issue with urs or not but try changing ur click to this :
agree = driver.find_element_by_xpath('//*[#id="introAgreeButton"]/span/span')
driver.execute_script("arguments[0].click();", agree)
The problem was that it didn't change frames here is the soulution code:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Chrome('C:\\Users\\gassp\\OneDrive\\Namizje\\Python.projects\\chromedriver.exe')
url = 'https://www.google.com/maps/'
driver.get(url)
WebDriverWait(driver, 10).until(EC.frame_to_be_available_and_switch_to_it((By.XPATH, '//*[#id="consent-bump"]/div/div[1]/iframe')))
agree = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//*[#id="introAgreeButton"]/span/span')))
agree.click()
#back to the main page
driver.switch_to_default_content()
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//*[#id="searchboxinput"]'))).send_keys('gostilne')
driver.submit()
I want to click on button on web site, but this button appears after you put your cursor on the right place, otherwise you can not see this button. I tried to ignore that and do just normal click by finding the element by xpath. But that did not work out, there was mistake:
selenium.common.exceptions.ElementNotInteractableException: Message: element not
interactable: element has zero size
How to deal with that?
To hover over the element you can use something like this:
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
chrome= webdriver.Chrome()
chrome.get('http://foo.bar')
element = chrome.find_element_by_css_selector("the_element_u_want")
hover = ActionChains(chrome).move_to_element(element)
hover.perform()
if the element takes time to appear use selenium's wait or time.sleep()
find more info about ActionChains here
this is some code I found here.
Usually, the ElementNotInteractableException is thrown when the element hasn't loaded completely. Here's a possible solution:
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as ec
driver = webdriver.Chrome('C:\\Users\\dell\\Libs\\chromedriver.exe')
driver.get("http://url")
driver.maximize_window()
# wait for element to appear, then hover it
wait = WebDriverWait(driver, 10)
men_menu = wait.until(ec.visibility_of_element_located((By.XPATH, "//a[#data-tracking-id='men']")))
ActionChains(driver).move_to_element(men_menu).perform()
This method worked well for me.
def hoverXPATH(xpath):
item = WebDriverWait(driver, 10)\
.until(EC.visibility_of_element_located((By.XPATH, xpath)))
ActionChains(driver).move_to_element(item).perform()
I am pretty new in selenium and I was looking forward to execute a click() button that expands a table, and after that I was looking forward to scrape the content of the expanded table.
This is what I tried:
url = 'https://www.telegraph.co.uk/markets-hub/assets/shares/'
phantomjs_path = '/usr/local/bin/phantomjs'
driver = webdriver.PhantomJS(executable_path=phantomjs_path,service_args=['--load-images=no'])
driver.get(url)
element = driver.find_element_by_xpath("//div[#class='tmg-show-more']")
html_source = driver.page_source
element.click()
driver.quit()
soup = BeautifulSoup(html_source,"html5lib")
tables=soup.find('table',class_="table-static kurser-table")
link=pd.read_html(str(tables),flavor='html5lib',thousands='.',decimal=',',header=0)
print(link)
The current output is as it follows:
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.ElementNotVisibleException: Message: {"errorMessage":"Element is not currently visible and may not be manipulated",
"request":{"headers":{"Accept":"application/json","Accept-Encoding":"identity","Connection":"close","Content-Length":"81","Content-
Type":"application/json;charset=UTF-8","Host":"127.0.0.1:58447","User-Agent":"Python http auth"},
"httpVersion":"1.1","method":"POST","post":"{\"id\": \":wdc:1556893378248\", \"sessionId\": \"f223cb80-6dae-11e9-b00b-4bb158952171\"}",
"url":"/click","urlParsed":{"anchor":"","query":"","file":"click","directory":"/","path":"/click","relative":"/click","port":"","host":"","password":"","user":"",
"userInfo":"","authority":"","protocol":"","source":"/click","queryKey":{},"chunks":["click"]},"urlOriginal":"/session/f223cb80-6dae-11e9-b00b-4bb158952171/element/:wdc:1556893378248/click"}}
Screenshot: available via screen
You would need to scroll down to let the selenium web driver knows where the element is actually present.
Code :
driver.get("https://www.telegraph.co.uk/markets-hub/assets/shares/")
wait = WebDriverWait(driver, 10)
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, 'div.section-box.kurser-table-container')))
element = driver.find_element_by_xpath("//a[#ng-click='stocksShowMore()']")
actions = ActionChains(driver)
actions.move_to_element(element).perform()
element.click()
Imports :
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
You are getting ElementNotVisibleException means you are trying to click the element before it is visible.
Also, you are getting the page source before clicking the button. You need to get the source after clicking and loading the new data.
Try the following code before getting the page source.
wait = WebDriverWait(driver, 20)
element = wait.until(EC.element_to_be_clickable(driver.find_element_by_xpath("//div[#class='tmg-show-more']")))
element.click()
wait.until(EC.invisibility_of_element(element))
To use the WebDriverWait you need to import these
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
I am attempting to click the "View More Results" button on the following page: http://www.chadbourne.com/search/people?az[b]=b
My code is straight forward, and I have tried a number of iterations.
driver = driver.Firefox()
driver.get("http://www.chadbourne.com/search/people?az[b]=b")
element = driver.find_element_by_partial_link_text("View more results")
or
element = driver.find_element_by_partial_link_text("view")
or
element = driver.find_element_by_partial_link_text("results")
No matter which of the above options I try, I get a NoSuchElementException.
This is odd, because the element clearly exists on the page:
View more results
Any thoughts?
It takes some time for the page to load, the element is not immediately available. Let's wait for it to be clickable and change the locator to a CSS selector:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Firefox()
driver.get("http://www.chadbourne.com/search/people?az[b]=b")
wait = WebDriverWait(driver, 20)
# get more results
more_results = wait.until(
EC.element_to_be_clickable((By.CSS_SELECTOR, ".load-more-pager-wrapper .pager-next"))
)
more_results.click()
I am trying to scrape an airbnb listing. I cant figure out a way to get the full list of amenities other than clicking on "more". I am using selenium to simulate the click, but it does nt seem to work.
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
url = 'https://www.airbnb.com/rooms/4660676'
driver = webdriver.Firefox()
driver.get(url)
elem = driver.find_element_by_xpath('//a[#class="expandable-trigger-more"]')
actions.click(elem).perform()
The XPath itself is correct, but you have not defined the actions:
from selenium.webdriver.common.action_chains import ActionChains
elem = driver.find_element_by_xpath('//a[#class="expandable-trigger-more"]')
actions = ActionChains(driver)
actions.click(elem).perform()
Note that you can simply use the WebElement's click() method instead:
elem = driver.find_element_by_xpath('//a[#class="expandable-trigger-more"]')
elem.click()
Works for me.
If you are getting NoSuchElementException error, you might need to wait for the link to be clickable via WebDriverWait and element_to_be_clickable.
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
element = WebDriverWait(driver, 10).until(
EC.element_to_be_clickable((By.XPATH, '//a[#class="expandable-trigger-more"]'))
)
element.click()
A very simple way of achieving this is given below
driver.find_element_by_xpath('//a[#class="expandable-trigger-more"]').click()
It works for me hope will work for you as well.