We Keep Coding

Selenium / Use pagination on site?

i want to trigger the pagination on this site:
https://www.kicker.de/bundesliga/topspieler/2008-09
I found the element with this XPATH in the chrome-inspector:
driver.find_element(By.XPATH,"//a[#class='kick__pagination__button kick__icon-Pfeil04 kick__pagination--icon']").click()
Now i want to click this element to go one page further - but i get an error.
This is my code:
import time
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from sys import platform
import os, sys
from datetime import datetime, timedelta
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from webdriver_manager.chrome import ChromeDriverManager
from fake_useragent import UserAgent
if __name__ == '__main__':
print(f"Checking chromedriver...")
os.environ['WDM_LOG_LEVEL'] = '0'
ua = UserAgent()
userAgent = ua.random
options = Options()
options.add_argument('--headless')
options.add_experimental_option ('excludeSwitches', ['enable-logging'])
options.add_experimental_option("prefs", {"profile.default_content_setting_values.notifications": 1})
options.add_argument("--disable-infobars")
options.add_argument("--disable-extensions")
options.add_argument("start-maximized")
options.add_argument('window-size=1920x1080')
options.add_argument('--no-sandbox')
options.add_argument('--disable-gpu')
options.add_argument(f'user-agent={userAgent}')
srv=Service(ChromeDriverManager().install())
driver = webdriver.Chrome (service=srv, options=options)
waitWebDriver = WebDriverWait (driver, 10)
seasonList = ["2008-09","2009-10","2010-11","2011-12","2012-13","2013-14","2014-15",
"2015-16","2016-17","2017-18","2018-19","2020-21", "2021-22"]
for season in seasonList:
tmpSeason = f"{season[:4]}/20{season[5:]}"
link = f"https://www.kicker.de/bundesliga/topspieler/{season}"
print(f"Working for link {link}...")
driver.get (link)
time.sleep(WAIT)
while True:
soup = BeautifulSoup (driver.page_source, 'html.parser')
tmpTABLE = soup.find("table")
tmpTR = tmpTABLE.find_all("tr")
driver.find_element(By.XPATH,"//a[#class='kick__pagination__button kick__icon-Pfeil04 kick__pagination--icon']").click()
time.sleep(WAIT)
But i get this error:
Traceback (most recent call last):
File "C:\Users\Polzi\Documents\DEV\Fiverr\ORDER\fireworkenter\collGrades.py", line 116, in <module>
driver.find_element(By.XPATH,"//a[#class='kick__pagination__button kick__icon-Pfeil04 kick__pagination--icon']").click()
File "C:\Users\Polzi\Documents\DEV\.venv\NormalScraping\lib\site-packages\selenium\webdriver\remote\webelement.py", line 80, in click
self._execute(Command.CLICK_ELEMENT)
File "C:\Users\Polzi\Documents\DEV\.venv\NormalScraping\lib\site-packages\selenium\webdriver\remote\webelement.py", line 693, in _execute
return self._parent.execute(command, params)
File "C:\Users\Polzi\Documents\DEV\.venv\NormalScraping\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 400, in execute
self.error_handler.check_response(response)
File "C:\Users\Polzi\Documents\DEV\.venv\NormalScraping\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 236, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.ElementNotInteractableException: Message: element not interactable
(Session info: headless chrome=99.0.4844.82)
How can i go to the next page using selenium?

The go to the next page you can click on the next page element inducing WebDriverWait for the element_to_be_clickable() and you can use either of the following locator strategies:
Using CSS_SELECTOR:
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "a.kick__pagination__button--active +a"))).click()
Using XPATH:
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//a[contains(#class, 'kick__pagination__button--active')]//following::a[1]"))).click()
Note: You have to add the following imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC

Python Selenium checking checkbox: selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element

I have written code to select the checkbox at following website: https://www.theatlantic.com/do-not-sell-my-personal-information/
I have tried following versions:
Version 1:
ele = driver.find_element_by_id('residency')
driver.execute_script("arguments[0].click()",ele)
Version 2: checkBox1 = driver.find_element_by_css_selector("input[id='residency']")
Version 3: driver.find_element_by_xpath("//input[#type='checkbox']")
However, for all of these versions I get following error:
Traceback (most recent call last):
File "website-functions/theatlantic.py", line 43, in <module>
atlantic_DD_formfill(california_resident, email, zipcode)
File "website-functions/theatlantic.py", line 30, in atlantic_DD_formfill
driver.find_element_by_xpath("//input[#type='checkbox']")
File "/usr/local/lib/python3.6/dist-packages/selenium/webdriver/remote/webdriver.py", line 394, in find_element_by_xpath
return self.find_element(by=By.XPATH, value=xpath)
File "/usr/local/lib/python3.6/dist-packages/selenium/webdriver/remote/webdriver.py", line 978, in find_element
'value': value})['value']
File "/usr/local/lib/python3.6/dist-packages/selenium/webdriver/remote/webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "/usr/local/lib/python3.6/dist-packages/selenium/webdriver/remote/errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"xpath","selector":"//input[#type='checkbox']"}
(Session info: headless chrome=80.0.3987.87)
Here you can see the full code:
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
import os
import time
def atlantic_DD_formfill(california_resident, email, zipcode):
chrome_options = Options()
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
chrome_options.add_experimental_option('useAutomationExtension', False)
driver = webdriver.Chrome(options=chrome_options)
driver.set_page_load_timeout(10)
driver.set_window_size(1124, 850) # set browser size
# link to data delete form
print("opening data delete form")
driver.get("https://www.theatlantic.com/do-not-sell-my-personal-information/")
#Select California Resident Field:
#ele = driver.find_element_by_id('residency')
#driver.execute_script("arguments[0].click()",ele)
#checkBox1 = driver.find_element_by_css_selector("input[id='residency']")
#if(NOT(checkBox1.isSelected())):
# checkBox1.click()
driver.find_element_by_xpath("//input[#type='checkbox']")
print("California Resident Field selected")
driver.find_element_by_id("email").send_keys(email)
driver.find_element_by_id("zip-code").send_keys(email)
# KEEP THIS DISABLED BC IT ACTUALLY SUBMITS
# driver.find_element_by_id("SubmitButton2").send_keys(Keys.ENTER)
print("executed")
time.sleep(4)
driver.quit()
return None
california_resident=True
email = "joe#musterman.com"
zipcode=12345
atlantic_DD_formfill(california_resident, email, zipcode)

There is an iframe present on the page, so you need to first switch to that iframe and then click on the element and as an another element is placed above the checkbox element, you need to use java script click method to click on the checkbox.
You can do it like:
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
import os
import time
def atlantic_DD_formfill(california_resident, email, zipcode):
chrome_options = Options()
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
chrome_options.add_experimental_option('useAutomationExtension', False)
driver = webdriver.Chrome(options=chrome_options)
driver.set_page_load_timeout(10)
driver.set_window_size(1124, 850) # set browser size
# link to data delete form
print("opening data delete form")
driver.get("https://www.theatlantic.com/do-not-sell-my-personal-information/")
driver.switch_to.frame(driver.find_element_by_tag_name('iframe'))
element = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH, "//input[#type='checkbox']")))
driver.execute_script("arguments[0].click();", element)

Python Selenium WebDriver unable to find element at all

Selenium webdriver has been unable to find any elements on page using different methods: class_name, id, & xpath.
Here's my code:
from selenium import webdriver
##from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.keys import Keys
import time
import random
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--disable-gpu")
driver = webdriver.Chrome(executable_path=r'C:\Users\acer\Downloads\chromedriver_win32\chromedriver.exe', chrome_options=chrome_options)
time.sleep(2)
driver.get('https://www.reddit.com/r/AskReddit/comments/fi04fh/what_are_some_spoilers_for_the_next_month_of_2020/')
time.sleep(2)
print(driver.title)
time.sleep(2)
element = driver.find_element_by_id("header")
print("done")
The title prints successfully but it fails on the line of driver.find_element_by_id("header").
In fact, I am trying to find the element whose class is "top-matter" (using find_by_class_name) but since this wasn't working, I tested it for other elements ("header") using respective methods ("xpath", "id") but nothing is working for me.
Can anyone provide some insight into the issue?
EDIT: Here's the error:
Traceback (most recent call last):
File "C:/Python34/reddit_test.py", line 20, in <module>
element = driver.find_element_by_id("header")
File "C:\Python34\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 269, in find_element_by_id
return self.find_element(by=By.ID, value=id_)
File "C:\Python34\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 752, in find_element
'value': value})['value']
File "C:\Python34\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 236, in execute
self.error_handler.check_response(response)
File "C:\Python34\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 192, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"id","selector":"header"}
(Session info: headless chrome=80.0.3987.132)
(Driver info: chromedriver=2.41.578737 (49da6702b16031c40d63e5618de03a32ff6c197e),platform=Windows NT 6.1.7601 SP1 x86_64)
Here's proof that the element exists...

in your url there is no header id
to ignore this exception
try this code:
from selenium import webdriver
##from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.keys import Keys
import time
import random
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--disable-gpu")
driver = webdriver.Chrome(executable_path=r'C:\Users\acer\Downloads\chromedriver_win32\chromedriver.exe', chrome_options=chrome_options)
time.sleep(2)
driver.get('https://www.reddit.com/r/AskReddit/comments/fi04fh/what_are_some_spoilers_for_the_next_month_of_2020/')
time.sleep(2)
print(driver.title)
time.sleep(2)
try:
element = driver.find_element_by_id("header")
except:
print("The Header isd dose not exist!")
exit()
print("done")
In Your Url The Header is dose not exist
You Can See By This Image!

The issue was that I use the old version of reddit whereas the default which Selenium and Hamza were opening is the new version which doesn't contain the elements I was trying to find

Scraping URLs with Python and selenium

I am trying to get a python selenium script working that should do the following:
Take text file, BookTitle.txt that is a list of Book Titles.
Using Python/Selenium then searches the site, GoodReads.com for that title.
Takes the URL for the result and makes a new .CSV file with column 1=book title and column 2=Site URL
I hope that we can get this working, then please help me with step by step to get it to run.
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.firefox.options import Options
from pyvirtualdisplay import Display
#from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common import keys
import csv
import time
import json
class Book:
def __init__(self, title, url):
self.title = title
self.url = url
def __iter__(self):
return iter([self.title, self.url])
url = 'https://www.goodreads.com/'
def create_csv_file():
header = ['Title', 'URL']
with open('/home/l/gDrive/AudioBookReviews/WebScraping/GoodReadsBooksNew.csv', 'w+', encoding='utf-8') as csv_file:
wr = csv.writer(csv_file, delimiter=',')
wr.writerow(header)
def read_from_txt_file():
lines = [line.rstrip('\n') for line in open('/home/l/gDrive/AudioBookReviews/WebScraping/BookTitles.txt', encoding='utf-8')]
return lines
def init_selenium():
chrome_options = Options()
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
options = Options()
options.add_argument('--headless')
global driver
driver = webdriver.Chrome("/home/l/gDrive/AudioBookReviews/WebScraping/chromedriver", chrome_options=chrome_options)
driver.get(url)
time.sleep(30)
driver.get('https://www.goodreads.com/search?q=')
def search_for_title(title):
search_field = driver.find_element_by_xpath('//*[#id="search_query_main"]')
search_field.clear()
search_field.send_keys(title)
search_button = driver.find_element_by_xpath('/html/body/div[2]/div[3]/div[1]/div[1]/div[2]/form/div[1]/input[3]')
search_button.click()
def scrape_url():
try:
url = driver.find_element_by_css_selector('a.bookTitle').get_attribute('href')
except:
url = "N/A"
return url
def write_into_csv_file(vendor):
with open('/home/l/gDrive/AudioBookReviews/WebScraping/GoodReadsBooksNew.csv', 'a', encoding='utf-8') as csv_file:
wr = csv.writer(csv_file, delimiter=',')
wr.writerow(list(vendor))
create_csv_file()
titles = read_from_txt_file()
init_selenium()
for title in titles:
search_for_title(title)
url = scrape_url()
book = Book(title, url)
write_into_csv_file(book)
Running the above, I get the following errors:
Traceback (most recent call last): File
"/home/l/gDrive/AudioBookReviews/WebScraping/GoodreadsScraper.py",
line 68, in
init_selenium() File "/home/l/gDrive/AudioBookReviews/WebScraping/GoodreadsScraper.py",
line 41, in init_selenium
driver = webdriver.Chrome("/home/l/gDrive/AudioBookReviews/WebScraping/chromedriver",
chrome_options=chrome_options) File
"/usr/local/lib/python3.6/dist-packages/selenium/webdriver/chrome/webdriver.py",
line 81, in init
desired_capabilities=desired_capabilities) File "/usr/local/lib/python3.6/dist-packages/selenium/webdriver/remote/webdriver.py",
line 157, in init
self.start_session(capabilities, browser_profile) File "/usr/local/lib/python3.6/dist-packages/selenium/webdriver/remote/webdriver.py",
line 252, in start_session
response = self.execute(Command.NEW_SESSION, parameters) File "/usr/local/lib/python3.6/dist-packages/selenium/webdriver/remote/webdriver.py",
line 321, in execute
self.error_handler.check_response(response) File "/usr/local/lib/python3.6/dist-packages/selenium/webdriver/remote/errorhandler.py",
line 242, in check_response
raise exception_class(message, screen, stacktrace) selenium.common.exceptions.WebDriverException: Message: unknown error:
Chrome failed to start: exited abnormally (unknown error:
DevToolsActivePort file doesn't exist) (The process started from
chrome location /usr/bin/google-chrome is no longer running, so
ChromeDriver is assuming that Chrome has crashed.) (Driver info:
chromedriver=2.44.609551
(5d576e9a44fe4c5b6a07e568f1ebc753f1214634),platform=Linux
4.15.0-60-generic x86_64)

There are couple of errors I cansee for now:
1) you have to uncomment chrome options and comment firefox' as you're passing the chromedriver later in code
# from selenium.webdriver.firefox.options import Options
from selenium.webdriver.chrome.options import Options
Btw, that pyvirtualdisplay is an alternative for headless chrome, you don't need it imported.
2) you have instantiated Options two times and you're using only the first one. Change your code to:
def init_selenium():
chrome_options = Options()
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.add_argument('--headless')
I guess these two are just for start, edit your question when you encounter the next problem you can't solve.

You are using chrome driver, but you comment it out at import.
from selenium.webdriver.chrome.options import Options
In the search function, the process is:
get page -> find search box -> input value -> enter keys -> grab results.
Something like this:
def search_for_title(title):
driver.get('https://www.goodreads.com/search?q=')
search_field = driver.find_element_by_name('q')
search_field.clear()
search_field.send_keys(title)
search_field.send_keys(keys.Keys.RETURN) # you missed this part
url = driver.find_element_by_xpath(
'/html/body/div[2]/div[3]/div[1]/div[2]/div[2]/table/tbody/tr[1]/td[2]/a')
print(url.get_attribute('href'))

How to start Chrome Browser through Chromedriver and Selenium

I am getting error issues all of a sudden with selenium and the chromedriver. I haven't changed a single thing yet I am met with these error messages. The script literally worked hours ago and now without any tweaks its not working.
traceback (most recent call last):
File "email.py", line 3, in <module>
from selenium import webdriver
File "C:\ProgramData\Anaconda3\lib\site-packages\selenium\webdriver\__init__.py", line 18, in <module>
from .firefox.webdriver import WebDriver as Firefox # noqa
File "C:\ProgramData\Anaconda3\lib\site-packages\selenium\webdriver\firefox\webdriver.py", line 20, in <module>
import http.client as http_client
File "C:\ProgramData\Anaconda3\lib\http\client.py", line 71, in <module>
import email.parser
File "C:\Users\Doe Labs\Desktop\Austin\Scripts\email.py", line 12, in <module>
options = webdriver.ChromeOptions()
Here is my corresponding code:
import pyautogui
import time
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.support.ui import WebDriverWait
caps = DesiredCapabilities().CHROME
#caps["pageLoadStrategy"] = "eager"
options = webdriver.ChromeOptions()
options.add_argument(r'load-extension=C:\Users\Doe Labs\Desktop\Austin\sales_prospecting\facebookpixelhelper')
#options.add_argument('start-fullscreen')
options.add_argument('disable-infobars')
driver=webdriver.Chrome(desired_capabilities = caps, executable_path=r'C:\Users\Doe Labs\Desktop\Austin\sales_prospecting\chromedriver', chrome_options=options)
driver.get('http://www.doelabs.com/')
driver.maximize_window()
Even more strange is that when open new terminal, load python, and type from selenium import webdriver, i dont get any errors. But, when I navigate to the folder where the script lives, and load python and type from selenium import webdriver, i get the error message that shows up above. I hope this can give some insight into my current predicament.

A few words about the solution :
email is a reserved word / keyword in Python Language, avoid using the word email within user defined filename/methods/classes.
pageLoadStrategy as eager is yet to be implemented in ChromeDriver, use either none or normal instead as per your requirement.
To maximize the Chrome Browser Window instead of maximize_window() use the argument start-maximized through ChromeOptions()
To load an extension use ChromeOptions as follows :
options.addExtensions(new File("/path/to/extension.crx"));
Here are the four methods to initialize Chrome Browser through ChromeDriver :
Vanila Method :
from selenium import webdriver
driver = webdriver.Firefox(r'C:\path\to\chromedriver.exe')
driver.get('http://www.doelabs.com/')
print("Page Title is : %s" %driver.title)
driver.quit()
Arguments as ChromeOptions :
from selenium import webdriver
options = webdriver.ChromeOptions()
options.addExtensions(new File("C:\Users\Doe Labs\Desktop\Austin\sales_prospecting\facebookpixelhelper.crx"));
options.add_argument('start-maximized')
options.add_argument('disable-infobars')
driver=webdriver.Chrome(chrome_options=options, executable_path=r'C:\path\to\chromedriver.exe')
driver.get('http://www.doelabs.com/')
print("Page Title is : %s" %driver.title)
driver.quit()
Capabilities as DesiredCapabilities :
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
caps = DesiredCapabilities().CHROME.copy()
caps["pageLoadStrategy"] = "normal"
driver = webdriver.Chrome(executable_path=r'C:\path\to\chromedriver.exe', desired_capabilities=caps)
driver.get('http://www.doelabs.com/')
print("Page Title is : %s" %driver.title)
driver.quit()
Arguments as ChromeOptions and Capabilities as DesiredCapabilities :
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
caps = DesiredCapabilities().CHROME.copy()
caps["pageLoadStrategy"] = "normal"
options = webdriver.ChromeOptions()
options.addExtensions(new File("C:\Users\Doe Labs\Desktop\Austin\sales_prospecting\facebookpixelhelper.crx"));
options.add_argument('start-maximized')
options.add_argument('disable-infobars')
driver=webdriver.Chrome(chrome_options=options, executable_path=r'C:\path\to\chromedriver.exe', desired_capabilities=caps)
driver.get('http://www.doelabs.com/')
print("Page Title is : %s" %driver.title)
driver.quit()

You might want to change
executable_path=r'C:\Users\Doe Labs\Desktop\Austin\sales_prospecting\chromedriver',
to
executable_path=r'C:\Users\Doe Labs\Desktop\Austin\sales_prospecting\chromedriver.exe',
You seem to have missed .exe, the extension of the executable file.