We Keep Coding

Python selenium find element returns nothing

I would like to find titles contain '募集说明书', but the following codes just return nothing.
No error, just nothing. empty results.
driver.get('http://www.szse.cn/disclosure/bond/notice/index.html')
wait = WebDriverWait(driver, 30)
datefield_st = wait.until(EC.element_to_be_clickable((By.XPATH, "//div[#class='input-group-wrap form-control dropdown-btn']/input[1]")))
datefield_st.click()
wait.until(EC.element_to_be_clickable((By.XPATH, "//div[#class='calendar-control'][1]//div[3]//a"))).click()
wait.until(EC.element_to_be_clickable((By.XPATH, "//ul[#class='monthselect'][1]//li[text()='{}']".format("1")))).click()
s1 = driver.find_element_by_class_name('input-left')
s1.send_keys("2022-1-1")
s2 = driver.find_element_by_class_name('input-right')
s2.send_keys("2022-1-18")
driver.find_element_by_id("query-btn").click()
while True:
time.sleep(2)
try:
links=[link.get_attribute('href') for link in wait.until(EC.presence_of_all_elements_located((By.XPATH,"//a[#attachformat][.//span[contains(text(),'募集说明书')]]")))]
titles=[title.text for title in wait.until(EC.visibility_of_all_elements_located((By.XPATH, "//span[#class='pull-left ellipsis title-text' and contains(text(), '募集说明书')]//parent::a")))]
dates=[date.text for date in wait.until(EC.visibility_of_all_elements_located((By.XPATH, "//span[#class='pull-left ellipsis title-text' and contains(text(), '募集说明书')]//ancestor::td//following-sibling::td")))]
for link, title, date in zip(links, titles, dates):
print(link,title,date)
<div class="text-title-box">
<a attachformat="pdf" attachpath="/disc/disk03/finalpage/2022-01-21/bb9854c5-9d89-4914-a6ea-219b487b874a.PDF" target="_blank" href="/disclosure/listed/bulletinDetail/index.html?bd5fd845-e810-42d3-98b3-d2501daaabc3" class="annon-title-link">
<span class="pull-left title-text multiline" title="22新资01：新疆金投资产管理股份有限公司2022年面向专业投资者公开发行公司债券（第一期）募集说明书">22新资01：新疆金投资产管理股份有限公司2022年面向专业投资者公开发行公司债券（第一期）募集说明书</span>
<span class="pull-left ellipsis title-icon" title="点击下载公告文件"><img src="http://res.static.szse.cn/modules/disclosure/images/icon_pdf.png">(5822k)</span>
<span class="titledownload-icon" title="点击下载公告文件"></span>
</a>
</div>
Could someone please help with this issue? Many thanks

Elements matching //a[#attachformat][.//span[contains(text(),'募集说明书')]] XPath are located on the bottom of presented search results, they are out of the visible screen until you scroll them into the view.
Also, you are using a wrong locators for titles. See my fixes there.
The same about dates.
Also there is no need to use wait.until(EC.visibility_of_all_elements_located 3 times there. Since elements are found (and scrolled into the view) you can simply get them by driver.find_elements.
I also see no need for while True: loop here, it will break your code from complete after getting those 2 elements data, but I leaved it as is since you mentioned you intend to click "next page" there.
from selenium.webdriver.common.action_chains import ActionChains
driver.get('http://www.szse.cn/disclosure/bond/notice/index.html')
wait = WebDriverWait(driver, 30)
actions = ActionChains(driver)
datefield_st = wait.until(EC.element_to_be_clickable((By.XPATH, "//div[#class='input-group-wrap form-control dropdown-btn']/input[1]")))
datefield_st.click()
wait.until(EC.element_to_be_clickable((By.XPATH, "//div[#class='calendar-control'][1]//div[3]//a"))).click()
wait.until(EC.element_to_be_clickable((By.XPATH, "//ul[#class='monthselect'][1]//li[text()='{}']".format("1")))).click()
s1 = driver.find_element_by_class_name('input-left')
s1.send_keys("2022-1-1")
s2 = driver.find_element_by_class_name('input-right')
s2.send_keys("2022-1-18")
driver.find_element_by_id("query-btn").click()
while True:
time.sleep(2)
try:
lower_link =wait.until(EC.presence_of_element_located((By.XPATH,"(//a[#attachformat][.//span[contains(text(),'募集说明书')]])[last()]")))
actions.move_to_element(lower_link).perform()
time.sleep(0.5)
links=[link.get_attribute('href') for link in driver.find_elements(By.XPATH,"//a[#attachformat][.//span[contains(text(),'募集说明书')]]")]
titles=[title.text for title in driver.find_elements(By.XPATH, "//span[contains(#class,'pull-left') and contains(text(), '募集说明书')]//parent::a")]
dates=[date.text for date in driver.find_elements(By.XPATH, "//span[contains(#class,'pull-left') and contains(text(), '募集说明书')]//ancestor::td//following-sibling::td")]
for link, title, date in zip(links, titles, dates):
print(link,title,date)

Python Selenium can't click on next page button

I want to click the next page until no more page, but it does not click.
returns the error:raise exception_class(message, screen, stacktrace)
StaleElementReferenceException: stale element reference: element is not attached to the page document
my codes:
Thanks in advance!
driver.get('http://www.chinamoney.com.cn/chinese/zjfxzx/?tbnm=%E6%9C%80%E6%96%B0&tc=null&isNewTab=1')
driver.implicitly_wait(10)
driver.refresh()
driver.implicitly_wait(10)
wait = WebDriverWait(driver, 5)
datefield_st = wait.until(EC.element_to_be_clickable((By.ID, "pdbp-date-1")))
datefield_st.click()
select_st = Select(wait.until(EC.element_to_be_clickable((By.CLASS_NAME, "ui-datepicker-year"))))
select_st.select_by_visible_text("2021")
select2 = Select(wait.until(EC.element_to_be_clickable((By.CLASS_NAME, "ui-datepicker-month"))))
select2.select_by_value("1")
day=1
wait.until(EC.element_to_be_clickable((By.XPATH, "//td[#data-handler='selectDay']/a[text()='{}']".format(str(day))))).click()
datefield_ed = wait.until(EC.element_to_be_clickable((By.ID, "pdbp-date-2")))
datefield_ed.click()
select_ed = Select(wait.until(EC.element_to_be_clickable((By.CLASS_NAME, "ui-datepicker-year"))))
select_ed.select_by_visible_text("2021")
select2 = Select(wait.until(EC.element_to_be_clickable((By.CLASS_NAME, "ui-datepicker-month"))))
select2.select_by_value("1")
day=1
wait.until(EC.element_to_be_clickable((By.XPATH, "//td[#data-handler='selectDay']/a[text()='{}']".format(str(day))))).click()
driver.find_element_by_link_text("查询").click()
while True:
driver.implicitly_wait(10)
links=[link.get_attribute('href') for link in driver.find_elements_by_xpath("//a[contains(#title,'同业存单') and not(contains(#title,'申购说明')) and not(contains(#title,'公告'))]")]
titles = [title.text for title in driver.find_elements_by_xpath("//a[contains(#title,'中期票据') and not(contains(#title,'申购说明')) and not(contains(#title,'公告'))]")]
dates = [date.text for date in driver.find_elements_by_xpath('//*[#class="san-grid-r text-date"]')]
driver.implicitly_wait(10)
for link, title,date in zip(links, titles,dates):
dataframe = pd.DataFrame({'col1':date,'col2':title,'col3':link},index=[0])
dataframe.to_csv('Chinamoney.csv',mode='a+',header=False,index=False,encoding='utf-8-sig')
print(link,title,date)
try:
driver.find_element_by_xpath('//*[contains(#class, "page-next")]').click()
except:
print('No more pages')

You passed two class names into selector while it's not allowed for search by class name. Either try
(By.CLASS_NAME, 'page-next')
or
(By.CSS_SELECTOR, '.page-btn.page-next')
Also your element and icon select the same element. So you don't need to define icon. Simply use element.click()

You are using:
driver.find_element_by_xpath('//*[contains(#class, "page-next")]').click()
Try:
element = driver.find_element_by_xpath('//*[contains(#class, "page-next")]')
driver.execute_script("arguments[0].click();", element)
If this doesnt work, you can try to obtain the url/link value and store it, and later you can go to the url or do what you want without click in it.

Scraping with selenium and BeautifulSoup doesn´t return all the items in the page

So I came from the question here
Now I am able to interact with the page, scroll down the page, close the popup that appears and click at the bottom to expand the page.
The problem is when I count the items, the code only returns 20 and it should be 40.
I have checked the code again and again - I'm missing something but I don't know what.
See my code below:
from selenium import webdriver
from bs4 import BeautifulSoup
import pandas as pd
import time
import datetime
options = webdriver.ChromeOptions()
options.add_argument('--ignore-certificate-errors')
options.add_argument('--incognito')
#options.add_argument('--headless')
driver = webdriver.Chrome(executable_path=r"C:\\chromedriver.exe", options=options)
url = 'https://www.coolmod.com/componentes-pc-procesadores?f=375::No'
driver.get(url)
iter=1
while True:
scrollHeight = driver.execute_script("return document.documentElement.scrollHeight")
Height=10*iter
driver.execute_script("window.scrollTo(0, " + str(Height) + ");")
if Height > scrollHeight:
print('End of page')
break
iter+=1
time.sleep(3)
popup = driver.find_element_by_class_name('confirm').click()
time.sleep(3)
ver_mas = driver.find_elements_by_class_name('button-load-more')
for x in range(len(ver_mas)):
if ver_mas[x].is_displayed():
driver.execute_script("arguments[0].click();", ver_mas[x])
time.sleep(10)
page_source = driver.page_source
soup = BeautifulSoup(page_source, 'lxml')
# print(soup)
items = soup.find_all('div',class_='col-xs-12 col-sm-6 col-sm-6 col-md-6 col-lg-3 col-product col-custom-width')
print(len(items))
````=
What is wrong?. I newbie in the scraping world.
Regards

Your while and for statements don't work as intended.
Using while True: is a bad practice
You scroll until the bottom - but the button-load-more button isn't displayed there - and Selenium will not find it as displayed
find_elements_by_class_name - looks for multiple elements - the page has only one element with that class
if ver_mas[x].is_displayed(): if you are lucky this will be executed only once because the range is 1
Below you can find the solution - here the code looks for the button, moves to it instead of scrolling, and performs a click. If the code fails to found the button - meaning that all the items were loaded - it breaks the while and moves forward.
url = 'https://www.coolmod.com/componentes-pc-procesadores?f=375::No'
driver.get(url)
time.sleep(3)
popup = driver.find_element_by_class_name('confirm').click()
iter = 1
while iter > 0:
time.sleep(3)
try:
ver_mas = driver.find_element_by_class_name('button-load-more')
actions = ActionChains(driver)
actions.move_to_element(ver_mas).perform()
driver.execute_script("arguments[0].click();", ver_mas)
except NoSuchElementException:
break
iter += 1
page_source = driver.page_source
soup = BeautifulSoup(page_source, 'lxml')
# print(soup)
items = soup.find_all('div', class_='col-xs-12 col-sm-6 col-sm-6 col-md-6 col-lg-3 col-product col-custom-width')
print(len(items))

Scraping next pages issue with selenium

I am trying to scrape basic information on google. The code that I am using is the following. Unfortunately it does not move to the next page and I am not figuring the reason why. I am using selenium and google chrome as browser (no firefox). Could you please tell me what is wrong in my code?
driver.get('https://www.google.com/advanced_search?q=google&tbs=cdr:1,cd_min:3/4/2020,cd_max:3/4/2020&hl=en')
search = driver.find_element_by_name('q')
search.send_keys('tea')
search.submit()
soup = BeautifulSoup(driver.page_source,'lxml')
result_div = soup.find_all('div', attrs={'class': 'g'})
titles = []
while True:
next_page_btn =driver.find_elements_by_xpath("//a[#id='pnnext']")
for r in result_div:
if len(next_page_btn) <1:
print("no more pages left")
break
else:
try:
title = None
title = r.find('h3')
if isinstance(title,Tag):
title = title.get_text()
print(title)
if title != '' :
titles.append(title)
except:
continue
element =WebDriverWait(driver,5).until(expected_conditions.element_to_be_clickable((By.ID,'pnnext')))
driver.execute_script("return arguments[0].scrollIntoView();", element)
element.click()

I set q in the query string to be an empty string. Used as_q not q for the search box name. And reordered your code a bit. I put a page limit in to stop it going on forever.
from selenium import webdriver
from bs4 import BeautifulSoup
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
driver = webdriver.Chrome()
driver.get('https://www.google.com/advanced_search?q=&tbs=cdr:1,cd_min:3/4/2020,cd_max:3/4/2020&hl=en')
search = driver.find_element_by_name('as_q')
search.send_keys('tea')
search.submit()
titles = []
page_limit = 5
page = 0
while True:
soup = BeautifulSoup(driver.page_source, 'lxml')
result_div = soup.find_all('div', attrs={'class': 'g'})
for r in result_div:
for title in r.find_all('h3'):
title = title.get_text()
print(title)
titles.append(title)
next_page_btn = driver.find_elements_by_id('pnnext')
if len(next_page_btn) == 0 or page > page_limit:
break
element = WebDriverWait(driver, 5).until(expected_conditions.element_to_be_clickable((By.ID, 'pnnext')))
driver.execute_script("return arguments[0].scrollIntoView();", element)
element.click()
page = page + 1
driver.quit()

Extract iFrame content using Selenium and BeautifulSoup

Sorry, this is my first post, so forgive me for all that I don't know yet! Thanks.
I am trying to complete the following form and extract the associating premium. When I run my code I would expect the annual premium of $156 to be extracted, but all I get is "Annual Premium: -"
Here is the code:
from selenium import webdriver
import time
from bs4 import BeautifulSoup
import requests
driver = webdriver.Chrome(r"C:\Users\tomwp\Downloads\chromedriver_win32\chromedriver.exe")
page = driver.get("https://www.earthquakeauthority.com/")
xpath = '//*[#id="form"]/header/div[2]/a'
btn = driver.find_element_by_xpath(xpath)
btn.click()
time.sleep(5)
iframe = driver.find_element_by_xpath("//iframe[#id='premiumCalc-iframe']")
driver.switch_to.frame(iframe)
xpath = '//*[#id="cea-page-1"]/div/div/div[1]/div/button[1]'
btn = driver.find_element_by_xpath(xpath)
btn.click()
xpath = '//*[#id="startdate"]'
incept_date = driver.find_element_by_xpath(xpath)
incept_date.send_keys("03/24/2019")
xpath = '//*[#id="participatingInsurer"]'
insurance_company = driver.find_element_by_xpath(xpath)
insurance_company.send_keys("Other")
xpath = '//*[#id="street"]'
street_address = driver.find_element_by_xpath(xpath)
street_address.send_keys("26 Los Indios")
xpath = '//*[#id="zipcode"]'
zip_code = driver.find_element_by_xpath(xpath)
zip_code.send_keys("92618")
xpath = '//*[#id="form-views"]/div[18]/div/button'
btn = driver.find_element_by_xpath(xpath)
btn.click()
xpath = '//*[#id="yearbuilt"]'
year_built = driver.find_element_by_xpath(xpath)
year_built.send_keys("2011")
xpath = '//*[#id="insuredvalue"]'
year_built = driver.find_element_by_xpath(xpath)
year_built.send_keys("100000")
xpath = '//*[#id="numberOfStories"]'
number_stories = driver.find_element_by_xpath(xpath)
number_stories.send_keys("Greater than one")
xpath = '//*[#id="foundationtype"]'
foundation = driver.find_element_by_xpath(xpath)
foundation.send_keys("slab")
xpath = '//*[#id="form-views"]/div[14]/div/button'
btn = driver.find_element_by_xpath(xpath)
btn.click()
soup = BeautifulSoup(driver.page_source, 'lxml')
premium = soup.find('div', class_='gauge-subtitle ng-binding ng-scope')
print(premium.text)
This is the $156 I would like to extract:
<div ng-if="isQuoting == false" class="gauge-subtitle ng-binding ng-scope">Annual Premium: $156.00</div>
Note that iframe id is as follows (not sure if this is helpful):
<iframe id="premiumCalc-iframe" style="width: 100%; border: none; height: 1397px;" scrolling="no" src="//calc.earthquakeauthority.com/app/index.html" cd_frame_id_="d0b3a5bcdcfe60ced66a29d282ad86c6"></iframe>
enter image description here

I have tried to make it a little more robust by adding in wait conditions. Your final quote page refreshes at the last click so you likely got a stale element exception. If you can find a decent indicator of that refresh completing you should replace my current time.sleep.
Personally, I would use CSS selectors throughout but I am sticking with xpath to align with your code.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Chrome(r"C:\Users\tomwp\Downloads\chromedriver_win32\chromedriver.exe")
page = driver.get("https://www.earthquakeauthority.com/")
xpath = '//*[#id="form"]/header/div[2]/a'
btn = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, xpath)))
btn.click()
iframe = WebDriverWait(driver,5).until(EC.presence_of_element_located((By.XPATH, "//iframe[#id='premiumCalc-iframe']")))
driver.switch_to.frame(iframe)
xpath = '//*[#id="cea-page-1"]/div/div/div[1]/div/button[1]'
btn = WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.XPATH, xpath)))
btn.click()
xpath = '//*[#id="startdate"]'
incept_date = WebDriverWait(driver,5).until(EC.presence_of_element_located((By.XPATH, xpath)))
incept_date.send_keys("03/24/2019")
xpath = '//*[#id="participatingInsurer"]'
insurance_company = driver.find_element_by_xpath(xpath)
insurance_company.send_keys("Other")
xpath = '//*[#id="street"]'
street_address = driver.find_element_by_xpath(xpath)
street_address.send_keys("26 Los Indios")
xpath = '//*[#id="zipcode"]'
zip_code = driver.find_element_by_xpath(xpath)
zip_code.send_keys("92618")
xpath = '//*[#id="form-views"]/div[18]/div/button'
btn = WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.XPATH, xpath)))
btn.click()
xpath = '//*[#id="yearbuilt"]'
year_built = WebDriverWait(driver,5).until(EC.presence_of_element_located((By.XPATH, xpath)))
year_built.send_keys("2011")
xpath = '//*[#id="insuredvalue"]'
year_built = driver.find_element_by_xpath(xpath)
year_built.send_keys("100000")
xpath = '//*[#id="numberOfStories"]'
number_stories = driver.find_element_by_xpath(xpath)
number_stories.send_keys("Greater than one")
xpath = '//*[#id="foundationtype"]'
foundation = driver.find_element_by_xpath(xpath)
foundation.send_keys("slab")
xpath = '//*[#id="form-views"]/div[14]/div/button'
btn = WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.XPATH, xpath)))
btn.click()
time.sleep(2)
quote = driver.find_element_by_css_selector(".gauge-subtitle").text
print(quote)

if I get you correct, you are able to navigate upto estimation page and able to see estimated annual premium value.
If thats the case then just try this code:
iframe = driver.find_element_by_xpath("//iframe[#id='premiumCalc-iframe']")
yourResult = driver.find_element_by_class_name("gauge-subtitle ng-binding ng-scope").text