import time
import sys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.options import Options
options = webdriver.ChromeOptions()
options.add_argument('--headless')
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
driver=webdriver.Chrome('chromedriver',options=options)
driver.get("https://rentry.co/wftw8/edit")
driver.implicitly_wait(20)
#print (driver.page_source)
try:
# here I selected the **span** element that I talk above
span = driver.find_element_by_xpath('//*[#id="text"]/div/div[5]/div[1]/div/div/div/div[5]/pre/span')
# change the innerText thwough js
driver.execute_script('arguments[0].innerText="Hello boy"', span)
# just wait for the id_edit_code to be present
edit = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.ID, "id_edit_code")))
edit.send_keys("iRfiNq6M")
# and same as you had it
#driver.find_element_by_id("submitButton").send_keys(Keys.ENTER)
#driver.find_element_by_link_text("Save").click()
driver.find_element_by_id("submitButton").click()
except:
print("Oops!", sys.exc_info()[0], "occurred.")
finally:
driver.close()
print("done")
There is no exception but still updating of the text is not reflecting in the url?
Even though there is timer which is enough for the whole code to get processed.
Then also there is no updation.
Currently the text is Hello if u visit the URL but I want it to be Hello boy using selenium , which is done by the below code line:
span = driver.find_element_by_xpath('//*[#id="text"]/div/div[5]/div[1]/div/div/div/div[5]/pre/span')
# change the innerText thwough js
driver.execute_script('arguments[0].innerText="Hello boy"', span)
But no updation!!?
Related
enter image description here
I want to make a web scraper for the news title on the news website, news.sky.com
The problem is, very often, the site will pop up a message to let me "accept all cookies" (see capture...)
I followed the guide on the similar question by using XPath. But, it feedbacks:
raise TimeoutException(message, screen, stacktrace)
Seemly, selenium can't find out the location based on XPath.
so, is it possible to click "Accept all"?
please help, thx.
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium import webdriver
from selenium.webdriver.edge.service import Service
from selenium.webdriver.common.by import By
e_driver_path = r"F:/Download/Portable Test/msedgedriver.exe"
# establish the web driver
s = Service(e_driver_path)
driver = webdriver.Edge(service=s)
driver.get("https://news.sky.com/uk")
# search = driver.find_element_by_id(ContentPlaceHolder1_NotifyBtn)
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//*[#id='notice']/div[3]/button[1]"))).click()
print("yes")
The cookie buttons are in an iframe on that page. Here is one way of clicking that button:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.keys import Keys
import time as t
chrome_options = Options()
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument('disable-notifications')
chrome_options.add_argument("window-size=1280,720")
webdriver_service = Service("chromedriver/chromedriver") ## path to where you saved chromedriver binary
driver = webdriver.Chrome(service=webdriver_service, options=chrome_options)
actions = ActionChains(driver)
wait = WebDriverWait(driver, 20)
url = "https://news.sky.com/uk"
driver.get(url)
wait.until(EC.frame_to_be_available_and_switch_to_it((By.XPATH, "//iframe[#title='SP Consent Message']")))
try:
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button[title='Accept all']"))).click()
print('accepted cookies')
except Exception as e:
print('no cookie button!')
driver.switch_to.default_content()
print('back to main content')
This will access the iframe, click the button, and then exit the iframe back to main content, and also print in terminal:
accepted cookies
back to main content
Selenium setup is chrome/chromedriver/linux, you just need to observe the import and the code after defining the driver, to adapt it to your own setup. Selenium docs can be found here: https://www.selenium.dev/documentation/
import selenium
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.remote import webelement
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import TimeoutException, NoSuchElementException
import time
url = "https://www.mrporter.com/en-gb/mens/product/nike/shoes/low-top-sneakers/space-hippie-04-recycled-stretch-knit-sneakers/19971654707345242"
PATH = 'C:\Program Files (x86)\chromedriver.exe'
browser = webdriver.Chrome(PATH)
browser.get(url)
element_dropdown = browser.find_element_by_class_name("CombinedSelect11__field CombinedSelect11__field--selectableOption CombinedSelect11__field--nativeSelect")
select = Select(element_dropdown)
try:
select.select_by_visible_text("8")
except NoSuchElementException:
print("the item doesnt exist")
I am trying to locate the dropdown menu of the link in my code. Once the dropdown box is located I want to search by visible text for a size 8. However whatever I try it still doesn't work.
You can try using explicit wait and then perform your operation. Please take a look at the below code which I have written to replicate your scenario. It's working fine for me. Do let me know if you face any problems.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
options = webdriver.ChromeOptions()
options.add_argument("start-maximized")
driver = webdriver.Chrome(options=options)
driver.get(
'https://www.mrporter.com/en-gb/mens/product/nike/shoes/low-top-sneakers/space-hippie-04-recycled-stretch-knit-'
'sneakers/19971654707345242')
wait = WebDriverWait(driver, 30)
wait.until(EC.visibility_of_element_located((By.XPATH, '//div[text()="Select a size"]'))).click()
wait.until(EC.visibility_of_element_located((By.XPATH, '//li[#data-value="8"]'))).click()
I need to scrape the image src from this popup. I have coded that but getting "AttributeError: 'NoneType' object has no attribute 'findElements'.
Here is the code.
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
from chromedriver_py import binary_path
import time
from time import sleep
url = 'https://www.macys.com/shop/product/black-tape-printed-contrast-trim-cardigan?ID=11398979&CategoryID=260&isDlp=true&swatchColor=Neutral%20Animal'
options = Options()
options.add_argument('--disable-blink-features=AutomationControlled')
driver = webdriver.Chrome(options=options, executable_path=binary_path)
wait = WebDriverWait(driver, 10)
driver.get(url)
sizechart_popup = wait.until(EC.element_to_be_clickable((By.XPATH, './/*[#class="sc-link"]'))).click()
sizechart = sizechart_popup.findElements(By.XPATH('.//*[#id="sizeImg"]/img')).get_attribute("src");
print(sizechart)
# Sleep of 10 seconds irrespective of whether element is present or not
time.sleep(50)
# Free up the resources
driver.close()
Thanks in advance
Try using all the available element identifiers if one doesn't work, it worked with css_selector.
sizechart = driver.find_element_by_css_selector('#sizeImg > img').get_attribute("src")
print(sizechart)
#Output:
https://assets.macysassets.com/dyn_img/size_charts/4011861.gif
I am trying to get a list of the prices from this page.
The class name of the elements I'm trying to get is called s-item__price.
This is my code:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
url = 'https://www.ebay.de/sch/i.html?_from=R40&_nkw=iphone+8+&_sacat=0&LH_TitleDesc=0&LH_ItemCondition=3000&rt=nc&LH_Sold=1&LH_Complete=1'
chrome_options = Options()
chrome_options.add_argument('--headless')
browser = webdriver.Chrome(options=chrome_options)
browser.get(url)
print(browser.find_elements_by_class_name('s-item__price'))
browser.quit()
The output is just an empty list.
You can use WebDriverWait to wait until the javascript generated the element:
wait = WebDriverWait(browser, 15) # 15 sec timeout
wait.until(expected_conditions.visibility_of_element_located((By.CLASS_NAME, 's-item__price')))
You could also use presence_of_elements_located but if it comes to click interaction it won't work with hidden elements.
So prefer using: visibility_of_element_located
Example Code:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
url = 'https://www.ebay.de/sch/i.html?_from=R40&_nkw=iphone+8+&_sacat=0&LH_TitleDesc=0&LH_ItemCondition=3000&rt=nc&LH_Sold=1&LH_Complete=1'
options = Options()
options.add_argument('--headless')
browser = webdriver.Chrome(options=options)
browser.get(url)
wait = WebDriverWait(browser, 15) # Throws a TimeoutException after 15 seconds
wait.until(expected_conditions.visibility_of_element_located((By.CLASS_NAME, 's-item__price')))
# you may also culd use the return value of the wait
print(browser.find_elements_by_class_name('s-item__price'))
browser.quit()
You get an empty list I think it because you need wait.
Use WebDriverWait and utilize .presence_of_all_elements_located to collect elements in a list.
Then extract them with a loop and you must call the .text method to grab the text
browser.get('https://www.ebay.de/sch/i.html?_from=R40&_nkw=iphone%208%20&_sacat=0&LH_TitleDesc=0&LH_ItemCondition=3000&rt=nc&LH_Sold=1&LH_Complete=1')
wait = WebDriverWait(browser, 20)
list_price = wait.until(EC.presence_of_all_elements_located((By.CLASS_NAME, 's-item__price')))
for price in list_price:
print(price.text)
driver.quit()
Following import:
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
zillow picture
I have the above image at https://www.zillow.com/homes/for_sale/7132668_zpid/globalrelevanceex_sort/60.780619,-65.522461,4.521666,-125.551758_rect/3_zm/
I cant seem to find the selector for tax history.
I tried to use driver wait but the table that is output is the price history not tax history.
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import NoSuchElementException
options = webdriver.ChromeOptions()
options.add_argument(f"user-agent={useragent[0]}")
options.add_argument('--proxy-server=%s' % ips[0])
options.add_argument('--incognito')
chromedriver = '~/Downloads/chromedriver'
chromedriver = os.path.expanduser(chromedriver)
driver = webdriver.Chrome(chromedriver, chrome_options=options)
driver.get('https://www.zillow.com/homes/for_sale/7132668_zpid/globalrelevanceex_sort/60.673178,-74.663086,4.653079,-116.323243_rect/3_zm/2_p/')
wait(driver, 10).until(EC.element_to_be_clickable((By.CLASS_NAME, "hdp-collapse"))).click()
table = wait(driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, "div#hdp-tax-history")))
Looks like you need some waits (and maybe some clicks to get that tab visible. You can write out the table. The below is just to show how you can access
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
url = 'https://www.zillow.com/homes/for_sale/7132668_zpid/globalrelevanceex_sort/53.566414,-73.081055,17.434511,-118.081055_rect/3_zm/'
d = webdriver.Chrome()
d.get(url)
WebDriverWait(d,20).until(EC.presence_of_element_located((By.ID , 'price-and-tax-history'))).click()
tabs = WebDriverWait(d,5).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".zsg-tab-link")))
tabs[1].click()
print(d.find_element_by_css_selector('#hdp-tax-history table').text) # just to show present