my process is there is a list of restaurants and i want to click on it and do some stuff and come back and same with the next restaurants in the list using a loop as you can see in below image
i followed this link but didn't get clarification
CODE
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
driver = webdriver.Chrome(executable_path='../chromedriver.exe')
driver.get("https://www.zomato.com/kolkata/dine-out?dishv2_id=76487754bd59c594cd5218d3427e68e0_2&rating_range=4.0-5.0")
screen_height = driver.execute_script("return window.screen.height;") # get the screen height of the web
i = 1
count = 0
scroll_pause_time = 1
while True:
# scroll one screen height each time
driver.execute_script("window.scrollTo(0, {screen_height}*{i});".format(screen_height=screen_height, i=i))
i += 1
time.sleep(scroll_pause_time)
# update scroll height each time after scrolled, as the scroll height can change after we scrolled the page
scroll_height = driver.execute_script("return document.body.scrollHeight;")
# Break the loop when the height we need to scroll to is larger than the total scroll height
if (screen_height) * i > scroll_height:
break
driver.execute_script("window.scrollTo(0, 0);")
res_list = []
titles = driver.find_elements_by_xpath('//a[#class="sc-dBAPYN kcrxQo"]')
#this is block of code which is error prone
for i in titles:
time.sleep(5)
driver.execute_script("window.scrollTo(0, 0);")
element = i.find_element_by_xpath('./div/h4')
driver.execute_script("arguments[0].click();", element)
time.sleep(3)
name_of_rests = driver.find_element_by_css_selector('#root > div > main > div > section.sc-kxynE.jzTfFZ > section > section > div > div > div > h1').text
res_list.append(name_of_rests)
driver.back()
print(res_list)
driver.close()
You have to define this
driver.find_elements_by_xpath('//a[#class="sc-dBAPYN kcrxQo"]')
again in loop.
Code :
number_of_titles = len(driver.find_elements_by_xpath('//a[#class="sc-dBAPYN kcrxQo"]'))
#this is block of code which is error prone
j = 0
for i in range(number_of_titles):
time.sleep(5)
titles = driver.find_elements_by_xpath('//a[#class="sc-dBAPYN kcrxQo"]')
driver.execute_script("window.scrollTo(0, 0);")
element = titles[j].find_element_by_xpath('.//div//h4')
driver.execute_script("arguments[0].click();", element)
time.sleep(3)
j = j +1
name_of_rests = driver.find_element_by_css_selector('#root > div > main > div > section.sc-kxynE.jzTfFZ > section > section > div > div > div > h1').text
res_list.append(name_of_rests)
driver.back()
Related
I am trying to scrape all the apps url from the target page:- https://play.google.com/store/apps?device= using the below code:-
from selenium import webdriver
from selenium.webdriver.chrome.service import Service as ChromeService
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager
import time
from tqdm import tqdm
options = webdriver.ChromeOptions()
options.add_argument('--no-sandbox')
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option("useAutomationExtension", False)
driver = webdriver.Chrome(ChromeDriverManager().install(), options=options)
driver.maximize_window()
items = ['phone','tablet','tv','chromebook','watch','car']
target_url = "https://play.google.com/store/apps?device="
all_apps = []
for cat in tqdm(items):
driver.get(target_url+cat)
time.sleep(2)
new_height = 0
last_height = 0
while True:
# Scroll down to bottom
driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
# Wait to load page
time.sleep(4)
# Calculate new scroll height and compare with last scroll height
new_height = driver.execute_script("return document.body.scrollHeight")
# break condition
if new_height == last_height:
break
last_height = new_height
for i in driver.find_elements(By.XPATH,"//a[contains(#href,'/store/apps/details')]"):
all_apps.append(i.get_attribute('href'))
The above code scrolls the page upside down and gives me the URLs of all the apps available on the page. However,
I tried to click the element using the below code but getting error:
driver.find_element(By.XPATH,"//i[contains(text(),'chevron_right')]").click()
error:-
ElementNotInteractableException: Message: element not interactable
(Session info: chrome=110.0.5481.77)
I tried using the below code:-
element = driver.find_element(By.XPATH,"//div[#class='bewvKb']") #any icon, may be that whatsapp icon here
hover = ActionChains(driver).move_to_element(element)
hover.perform()
element = driver.find_element(By.XPATH,"//i[text()='chevron_right']")
element.click()
There is no option to click on the highlighted button as shown in the image. Can anyone help me with this like how to scroll the page sideways so that all the contents can be scraped from the page?
The problem, that right arrow icon appears only when you hover upon any of the icon which is on the line. So first hover any of the icon, so that right arrow would appear and then issue the click, mostly like this
element = driver.find_element_by_css_selector("#yDmH0d > c-wiz.SSPGKf.glB9Ve > div > div > div.N4FjMb.Z97G4e > c-wiz > div > c-wiz > c-wiz:nth-child(1) > c-wiz > section > div > div > div > div > div > div.aoJE7e.b0ZfVe > div:nth-child(1) > div > div > a > div.TjRVLb > img") #any icon, may be that whatsapp icon here
hover = ActionChains(driver).move_to_element(element)
hover.perform()
element = driver.find_element_by_xpath("//i[text()='chevron_right']")
element.click()
I am trying to click all next pages until last page which it does successfully on this website. However, it reaches the last page and then waits indefinitely. How can I best achieve this script to then proceed to the rest of the script once it reaches the last page? I could do an explicit wait time of 15 seconds timeout but this feels very slow and not the best way of doing this. Thanks
Full code
i = 1
while i < 6:
try:
time.sleep(2)
#time.sleep(random, 3)
WebDriverWait(driver, 15).until(EC.element_to_be_clickable((By.CSS_SELECTOR, ".name:nth-child(1)")))
WebDriverWait(driver, 100).until(lambda driver: driver.execute_script('return document.readyState') == 'complete')
element = WebDriverWait(driver, 20).until(lambda driver: driver.find_element(By.CSS_SELECTOR, ".name:nth-child(1) , bf-coupon-table:nth-child(1) tr:nth-child(1) .matched-amount-value"))
scroll = driver.find_element(By.CSS_SELECTOR, ".coupon-page-navigation__label--next")
driver.execute_script("arguments[0].scrollIntoView();", scroll)
link = driver.find_element_by_css_selector('[href^=http://somelink.com/]')
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "bf-coupon-page-navigation > ul > li:nth-child(4) > a")))
NextStory = WebDriverWait(driver, 15).until(EC.element_to_be_clickable((By.CSS_SELECTOR, 'bf-coupon-page-navigation > ul > li:nth-child(4) > a')))
link = driver.find_element_by_css_selector('bf-coupon-page-navigation > ul > li:nth-child(4) > a')
NextStory.click()
except:
i = 6
You are not getting to the except block, otherwise the loop would stop.
I think that this line is causing the slowness:
driver.execute_script("arguments[0].scrollIntoView();", scroll)
That is bacause all your other actions are driver actions with timeouts. Try clicking on the webpage and then just click page down key in a loop:
from selenium.webdriver.common.keys import Keys
# Click on a static element in your page
for _ in range(100):
driver.send_keys(Keys.PAGE_DOWN)
Using Selenium to interact with authors on Medium.com
I am trying to target a popup element with selenium that appears when text is double-clicked. Once this is detected and clicked it opens a message box to the right. My end goal has been to insert (send_keys) text into this text box and yet it is proving to be quite difficult.
[the dynamic element]1 # the element on the far right is the button to open the chat box.
enter image description here2 # this is the text box
WHAT I HAVE TRIED:
from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.common.keys import Keys
def interact_with_author(message):
# variables
target_class = "meteredContent"
body_css = "body"
header_xpath = "/html/head"
second_article_css = "#root > div > div.s.n.t > div.ah.ay > div > div.n.p > div > div:nth-child(2)" # the second article on the page
first_par = "#\39 993"
second_par = "#\35 f50"
first_par_css = "#root > div > div.s > article > div > section > div > div > p"
first_par_class = "ht hu dj hv b ei hw hx hy el hz ia ib ic id ie if ig ih ii ij ik il im in io db eg"
wait_time = 5 # seconds to wait when sleep is called with the wait_time variable
#code
text_box = driver.find_element_by_css_selector('body > div:nth-child(47) > div > div > div > div > div')
action = ActionChains(driver)
listing=WebDriverWait(driver,10).until(EC.presence_of_all_elements_located((By.TAG_NAME,"a")))
articles = driver.find_elements_by_tag_name("a")
an_article = driver.find_element_by_css_selector(second_article_css)
an_article.click()
time.sleep(wait_time) # todo change to sleep four seconds after the article is fully loaded
listing=WebDriverWait(driver,10).until(EC.presence_of_all_elements_located((By.TAG_NAME,"p")))
try:
paragraphs = driver.find_elements_by_tag_name('p')
driver.execute_script("document.body.style.zoom='250%'")
try:
first_par = ''
for i in range(1,len(paragraphs)):
first_par_commentable = None
try:
first_par_commentable = driver.find_element_by_xpath(f"/html/body/div[1]/div/div[3]/article/div/section/div/div/p[{i}]")
driver.execute_script("document.body.style.zoom='200%'")
except Exception as e:
ic(e)
if first_par_commentable != None:
break
except Exception as f:
ic(f)
try:
first_par_commentable.click()
action.double_click(first_par_commentable).perform()
time.sleep(random.randint(1,3))
except Exception as e:
ic(e)
except Exception as e:
ic(e)
'''
If anyone knows how to access this element quickly and in a scaleable way it would be appreciated.
'''
There are 41 category checkbox, 12 are visible and remain are hidden, need to click on show more to show
hidden checkboxes.
this code
do that:: 1-loop >> 2-click on show more if i <=12 >> 3-click on checkbox
# -*- coding: utf-8 -*-
import time
from selenium import webdriver
driver = webdriver.Chrome(executable_path='C:\\Users\\Compu City\\Desktop\\chromedriver.exe')
driver.get('https://www.amazon.com/international-sales-offers/b/?ie=UTF8&node=15529609011&ref_=nav_navm_intl_deal_btn')
time.sleep(10)
res = driver.execute_script("return document.documentElement.outerHTML")
i=0
while i <= 41 :
if(i <=12):
driver.find_element_by_css_selector('#widgetFilters > div:nth-child(1) > div.a-row.a-expander-container.a-expander-inline-container > a > span').click()
driver.find_elements_by_css_selector('.a-expander-container .a-checkbox label .a-label')[i].click()
driver.implicitly_wait(2)
time.sleep(2)
i+=1
the problem: code works well with 12 checkboxes and then click on show more then click just on the first checkbox NO.13 then get this error
D:\python neww>aaa.py
DevTools listening on ws://127.0.0.1:59799/devtools/browser/ac2188e0-10e3-493b-9d91-614731f8d135
Traceback (most recent call last):
File "D:\python neww\aaa.py", line 14, in <module>
driver.find_elements_by_css_selector('.a-expander-container .a-checkbox label .a-label')[i].click()
IndexError: list index out of range
i searched a lot about selution what i found that echeckbox not load , but when change .click() with .text
I get all checkbox label names, so how they don't load
i mage for checkboxes
main problem with css selectors because there are three structures for html
1-first structure for first 12 check box
2-second one for first checkbox after click show more
3-third one when click on first checkbox after show more all structure will change and will include all check box
(i didn't use third part because it will make code so long and i want to short the code)
so try this code , after click on checkbox you will need to click clear so i think this code will be useful
first_part = 12
while first_part <=12:
#click on category checkbox
driver.find_element_by_css_selector('#widgetFilters > div:nth-child(1) > div.a-row.a-expander-container.a-expander-inline-container > span:nth-child({}) > div > label'.format(first_part)).click()
driver.implicitly_wait(3)
time.sleep(3)
#clear filter
driver.find_element_by_css_selector('#widgetFilters > div:nth-child(1) > span > div > a').click()
driver.implicitly_wait(3)
time.sleep(3)
#print text of checkbox category
print(driver.find_element_by_css_selector('#widgetFilters > div:nth-child(1) > div.a-row.a-expander-container.a-expander-inline-container > span:nth-child({}) > div > label'.format(first_part)).text)
first_part+=1
second_part = 1
while second_part <= 26:
#show more
driver.find_element_by_css_selector('#widgetFilters > div:nth-child(1) > div.a-row.a-expander-container.a-expander-inline-container > a > span').click()
driver.implicitly_wait(2)
time.sleep(2)
#print text of checkbox category
print(driver.find_element_by_css_selector('#widgetFilters > div:nth-child(1) > div.a-row.a-expander-container.a-expander-inline-container > div > span:nth-child({}) > div > label'.format(second_part)).text)
#click on category checkbox
driver.find_element_by_css_selector('#widgetFilters > div:nth-child(1) > div.a-row.a-expander-container.a-expander-inline-container > div > span:nth-child({}) > div > label'.format(second_part)).click()
driver.implicitly_wait(3)
time.sleep(3)
#clear filter
driver.find_element_by_css_selector('#widgetFilters > div:nth-child(1) > span > div > a').click()
driver.implicitly_wait(3)
time.sleep(3)
second_part+=1
the problem with css selectors
try this code
i = 5
while i <= 29:
driver.implicitly_wait(3)
time.sleep(3)
if(i==5):
cat =driver.find_element_by_css_selector('#widgetFilters > div:nth-child(1) > div.a-row.a-expander-container.a-expander-inline-container > div > span:nth-child({}) > div > label'.format(i)).click()
driver.implicitly_wait(3)
time.sleep(3)
else:
y=11
while y<=29:
cat = driver.find_element_by_css_selector('#widgetFilters > div:nth-child(1) > span:nth-child({}) > div > label'.format(y)).click()
y+=1
driver.implicitly_wait(3)
time.sleep(3)
driver.implicitly_wait(3)
time.sleep(3)
i += 1
print('i: ', i)
Here is the simple and sweet script that will work.
driver.get('https://www.amazon.com/international-sales-offers/b/?ie=UTF8&node=15529609011&ref_=nav_navm_intl_deal_btn')
checkCSS = "[class='a-row a-spacing-small filterItem']:nth-of-type(1) .a-checkbox.checkbox.a-spacing-micro"
WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.CSS_SELECTOR, checkCSS)))
departments = len(driver.find_elements_by_css_selector(checkCSS))
for depNum in range(departments):
print(depNum)
if depNum == 12:
WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.CSS_SELECTOR, "span.a-expander-prompt"))).click()
time.sleep(1)
WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.CSS_SELECTOR, checkCSS)))
driver.find_elements_by_css_selector(checkCSS)[depNum].click()
Please help, I want to scroll down to end of the bage but it stops.
the code that i try is here
browser = webdriver.Chrome()
browser.get(url)
button = browser.find_element_by_tag_name("html")
old=""
new=" "
while len(new)>len(old):
old = browser.page_source
button.send_keys(Keys.END)
browser.implicitly_wait(40)
new = browser.page_source
your script for that
driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
you can set no. of scrolls needed to get full length
scrolls = 4
while True:
scrolls -= 1
driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
time.sleep(3)
if scrolls < 0:
break