We Keep Coding

I want to change an attribute and give it a value with selenium

I have 19 span with the same attribute data-checked
<span id="u25-accordion-panel--61" data-type="checkbox" data-checked style="display: none;"></span>
I want to change the attribute in all the spans, I tried with 1 but I get the following error
options = webdriver.EdgeOptions()
options.add_argument("start-maximized")
driver = webdriver.Edge(options=options)
driver.implicitly_wait(20)
driver.get(
'https://www.udemy.com/course/angular-10-fundamentos-8-app/')
elem = driver.find_element(By.CSS_SELECTOR,
'#udemy > div.ud-main-content-wrapper > div.ud-main-content > div > div > div.paid-course-landing-page__container > div.paid-course-landing-page__body > div > div:nth-child(3) > div > button')
elem.click()
elems = driver.find_elements(By.CSS_SELECTOR,
'div.accordion-panel--panel--24beS > span')
driver.execute_script("arguments[0].data-checked = 'checked';", elems[1])
driver.execute_script("arguments[0].data-checked = 'checked';", elems[1])
selenium.common.exceptions.JavascriptException: Message: javascript error: Invalid left-hand side in assignment
(Session info: MicrosoftEdge=109.0.1518.78)

To change set the value of data-checked attribute for all the accordion-panel elements as checked you can use the setAttribute() method as follows:
Code block:
driver.get('https://www.udemy.com/course/angular-10-fundamentos-8-app/')
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "#udemy > div.ud-main-content-wrapper > div.ud-main-content > div > div > div.paid-course-landing-page__container > div.paid-course-landing-page__body > div > div:nth-child(3) > div > button"))).click()
elements = driver.find_elements(By.CSS_SELECTOR, 'div.accordion-panel--panel--24beS > span')
for element in elements:
driver.execute_script("arguments[0].setAttribute('data-checked', 'checked')", element)
Browser Snapshot:

What is the correct element to use from this section of html?

I'm using selenium to automatically go to various webpages and download an XML file.
This has been working fine but it has suddenly stopped working, and it's due to selenium not being able to find the element.
I've tried using selector gadget with CSS & Xpath. I've tried directly copying from the inspect panel, I've tried using the wait function on selenium until the element fully shows and I'm getting no luck.
This is the html of where the download button is
<div class="video-playlist-xml" data-reactid=".2.0"><i class="icon-download-xml-green" data-toggle="tooltip" data-placement="top" title="" data-original-title="Download xml file of the match" data-reactid=".2.0.0.0"></i></div>
The download button is just above the video, above the scoreline.
for x in range(number_of_clicks):
driver = webdriver.Chrome(executable_path=r'C:\Users\James\OneDrive\Desktop\webdriver\chromedriver.exe',options = options)
driver.get('https://football.instatscout.com/teams/978/matches')
time.sleep(10)
print("Page Title is : %s" %driver.title)
WebDriverWait(driver,20).until(EC.element_to_be_clickable((By.CSS_SELECTOR,"#team-table1 > div.table-scroll-inner > div.team-stats-wrapper.team-stats-wrapper_no-vertical-scroll > table > tbody > tr:nth-child(" +str(x+1) + ") > td:nth-child(1) > div > div.styled__MatchPlay-sc-10ytjn2-1.hkIvhi > i"))).click()
WebDriverWait(driver,20).until(EC.element_to_be_clickable((By.CSS_SELECTOR,"#root > div > article > section.player-details > div > div.OutsideClickWrapper-sc-ktqo9u.cTxKts > div > a:nth-child(1) > span > span"))).click()
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div.video-playlist-xml > a[href] > i.icon-download-xml-green[data-original-title='Download xml file of the match']"))).click()
chks = driver.find_elements_by_css_selector("#players > div.control-block > div.control-block__container.control-block__container--large > button")
for chk in chks:
chk.click()
time.sleep(15)
driver.quit()
error Traceback
---------------------------------------------------------------------------
TimeoutException Traceback (most recent call last)
~\AppData\Local\Temp/ipykernel_26436/2420202332.py in <module>
6 WebDriverWait(driver,20).until(EC.element_to_be_clickable((By.CSS_SELECTOR,"#team-table1 > div.table-scroll-inner > div.team-stats-wrapper.team-stats-wrapper_no-vertical-scroll > table > tbody > tr:nth-child(" +str(x+1) + ") > td:nth-child(1) > div > div.styled__MatchPlay-sc-10ytjn2-1.hkIvhi > i"))).click()
7 WebDriverWait(driver,20).until(EC.element_to_be_clickable((By.CSS_SELECTOR,"#root > div > article > section.player-details > div > div.OutsideClickWrapper-sc-ktqo9u.cTxKts > div > a:nth-child(1) > span > span"))).click()
----> 8 WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div.video-playlist-xml > a[href] > i.icon-download-xml-green[data-original-title='Download xml file of the match']"))).click()
9 #WebDriverWait(driver,20).until(EC.element_to_be_clickable((By.XPATH,'//*[contains(concat( " ", #class, " " ), concat( " ", "video-playlist-xml", " " ))]//a | //*[contains(concat( " ", #class, " " ), concat( " ", "icon-download-xml-green", " " ))]'))).click()
10 chks = driver.find_elements_by_css_selector("#players > div.control-block > div.control-block__container.control-block__container--large > button")
~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\selenium\webdriver\support\wait.py in until(self, method, message)
78 if time.time() > end_time:
79 break
---> 80 raise TimeoutException(message, screen, stacktrace)
81
82 def until_not(self, method, message=''):
TimeoutException: Message:

To locate a clickable element instead of presence_of_element_located() you need to induce WebDriverWait for the element_to_be_clickable() and you can use either of the following Locator Strategies:
Using CSS_SELECTOR:
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div.video-playlist-xml > a[href] > i.icon-download-xml-green[data-original-title='Download xml file of the match']"))).click()
Using XPATH:
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//div[#class='video-playlist-xml']/a[#href]/i[#class='icon-download-xml-green' and #data-original-title='Download xml file of the match']"))).click()
Note: You have to add the following imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC

Stupidly, I've just realised that clicking one of the elements was opening a new window. Issues solved. Thanks guys.

How to create iterable loop in css selector

I got 25 links like this
driver.find_element_by_css_selector('.v-table-table > tbody > tr:nth-child(i) > td:nth-child(4) > div > div > a').click()
and try to click them all by:
for i in range (1,25):
link = driver.find_element_by_css_selector('.v-table-table > tbody > tr:nth-child(i) > td:nth-child(4) > div > div > a')
link.click()
print(link)
time.sleep(1)
So i think that something wrong in my code, cause i get an error =)
Traceback (most recent call last):
File "/Users/admin/Desktop/python_1pk/get_response.py", line 29, in <module>
driver.find_element_by_css_selector('.v-table-table > tbody > tr:nth-child(i) > td:nth-child(4) > div > div > a').click()
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/selenium/webdriver/remote/webdriver.py", line 598, in find_element_by_css_selector
return self.find_element(by=By.CSS_SELECTOR, value=css_selector)
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/selenium/webdriver/remote/webdriver.py", line 976, in find_element
return self.execute(Command.FIND_ELEMENT, {
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/selenium/webdriver/remote/webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/selenium/webdriver/remote/errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.InvalidSelectorException: Message: invalid selector: An invalid or illegal selector was specified
(Session info: chrome=89.0.4389.82)
screenshot

I assume there's an issue with your CSS selector.
You are trying to form it as '.v-table-table > tbody > tr:nth-child(i) > td:nth-child(4) > div > div > a', but in this notation, the i variable does not get substituted to it's value.
Try using an f-string:
selector = f".v-table-table > tbody > tr:nth-child({i}) > td:nth-child(4) > div > div > a"
Note the curly braces around i - it's literally "place the value of the i variable here"

As I don't know how your page look like I assume it will be like this stackoverflow pages :https://stackoverflow.com/questions/tagged/python?tab=newest&page=2&pagesize=15 .
I tried to click on every link to do so I used the full xpath (you can use normal I think). With the xpath you find where is the article number so you can create a for loop and cThere is a code I tried I hope it will help you :
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
driver = webdriver.Chrome(Your path to the chromedriver)
url = "https://stackoverflow.com/questions/tagged/python?tab=newest&page=2&pagesize=15"
driver.get(url)
time.sleep(5)
"/html/body/div[3]/div[2]/div[1]/div[5]/div[1]/div/div[2]/h3/a" #xpath of the first title page
"/html/body/div[3]/div[2]/div[1]/div[5]/div[15]/div/div[2]/h3/a" #xpath of the last title page
for k in range(1,16):
driver.find_element_by_xpath("/html/body/div[3]/div[2]/div[1]/div[5]/div["+str(k)+"]/div/div[2]/h3/a").click()
"/html/body/div[3]/div[2]/div[1]/div[5]/div[1]/div"
print("clicked")
time.sleep(2)
driver.get(url)lick on every article.

(python) list index out of range - with using selenium click event

There are 41 category checkbox, 12 are visible and remain are hidden, need to click on show more to show
hidden checkboxes.
this code
do that:: 1-loop >> 2-click on show more if i <=12 >> 3-click on checkbox
# -*- coding: utf-8 -*-
import time
from selenium import webdriver
driver = webdriver.Chrome(executable_path='C:\\Users\\Compu City\\Desktop\\chromedriver.exe')
driver.get('https://www.amazon.com/international-sales-offers/b/?ie=UTF8&node=15529609011&ref_=nav_navm_intl_deal_btn')
time.sleep(10)
res = driver.execute_script("return document.documentElement.outerHTML")
i=0
while i <= 41 :
if(i <=12):
driver.find_element_by_css_selector('#widgetFilters > div:nth-child(1) > div.a-row.a-expander-container.a-expander-inline-container > a > span').click()
driver.find_elements_by_css_selector('.a-expander-container .a-checkbox label .a-label')[i].click()
driver.implicitly_wait(2)
time.sleep(2)
i+=1
the problem: code works well with 12 checkboxes and then click on show more then click just on the first checkbox NO.13 then get this error
D:\python neww>aaa.py
DevTools listening on ws://127.0.0.1:59799/devtools/browser/ac2188e0-10e3-493b-9d91-614731f8d135
Traceback (most recent call last):
File "D:\python neww\aaa.py", line 14, in <module>
driver.find_elements_by_css_selector('.a-expander-container .a-checkbox label .a-label')[i].click()
IndexError: list index out of range
i searched a lot about selution what i found that echeckbox not load , but when change .click() with .text
I get all checkbox label names, so how they don't load
i mage for checkboxes

main problem with css selectors because there are three structures for html
1-first structure for first 12 check box
2-second one for first checkbox after click show more
3-third one when click on first checkbox after show more all structure will change and will include all check box
(i didn't use third part because it will make code so long and i want to short the code)
so try this code , after click on checkbox you will need to click clear so i think this code will be useful
first_part = 12
while first_part <=12:
#click on category checkbox
driver.find_element_by_css_selector('#widgetFilters > div:nth-child(1) > div.a-row.a-expander-container.a-expander-inline-container > span:nth-child({}) > div > label'.format(first_part)).click()
driver.implicitly_wait(3)
time.sleep(3)
#clear filter
driver.find_element_by_css_selector('#widgetFilters > div:nth-child(1) > span > div > a').click()
driver.implicitly_wait(3)
time.sleep(3)
#print text of checkbox category
print(driver.find_element_by_css_selector('#widgetFilters > div:nth-child(1) > div.a-row.a-expander-container.a-expander-inline-container > span:nth-child({}) > div > label'.format(first_part)).text)
first_part+=1
second_part = 1
while second_part <= 26:
#show more
driver.find_element_by_css_selector('#widgetFilters > div:nth-child(1) > div.a-row.a-expander-container.a-expander-inline-container > a > span').click()
driver.implicitly_wait(2)
time.sleep(2)
#print text of checkbox category
print(driver.find_element_by_css_selector('#widgetFilters > div:nth-child(1) > div.a-row.a-expander-container.a-expander-inline-container > div > span:nth-child({}) > div > label'.format(second_part)).text)
#click on category checkbox
driver.find_element_by_css_selector('#widgetFilters > div:nth-child(1) > div.a-row.a-expander-container.a-expander-inline-container > div > span:nth-child({}) > div > label'.format(second_part)).click()
driver.implicitly_wait(3)
time.sleep(3)
#clear filter
driver.find_element_by_css_selector('#widgetFilters > div:nth-child(1) > span > div > a').click()
driver.implicitly_wait(3)
time.sleep(3)
second_part+=1

the problem with css selectors
try this code
i = 5
while i <= 29:
driver.implicitly_wait(3)
time.sleep(3)
if(i==5):
cat =driver.find_element_by_css_selector('#widgetFilters > div:nth-child(1) > div.a-row.a-expander-container.a-expander-inline-container > div > span:nth-child({}) > div > label'.format(i)).click()
driver.implicitly_wait(3)
time.sleep(3)
else:
y=11
while y<=29:
cat = driver.find_element_by_css_selector('#widgetFilters > div:nth-child(1) > span:nth-child({}) > div > label'.format(y)).click()
y+=1
driver.implicitly_wait(3)
time.sleep(3)
driver.implicitly_wait(3)
time.sleep(3)
i += 1
print('i: ', i)

Here is the simple and sweet script that will work.
driver.get('https://www.amazon.com/international-sales-offers/b/?ie=UTF8&node=15529609011&ref_=nav_navm_intl_deal_btn')
checkCSS = "[class='a-row a-spacing-small filterItem']:nth-of-type(1) .a-checkbox.checkbox.a-spacing-micro"
WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.CSS_SELECTOR, checkCSS)))
departments = len(driver.find_elements_by_css_selector(checkCSS))
for depNum in range(departments):
print(depNum)
if depNum == 12:
WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.CSS_SELECTOR, "span.a-expander-prompt"))).click()
time.sleep(1)
WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.CSS_SELECTOR, checkCSS)))
driver.find_elements_by_css_selector(checkCSS)[depNum].click()

Remember information about the previous page?

I'm trying to scrape Bet365 for it's live soccer odds. I'm iterating over a list of live matches available. For each match I need to click on the match and I'll be directed to some new content where I have all the detailed oddsinfo. From here it crashes when I go back to keep on my iterating.
It throws this error:
Traceback (most recent call last):
File "/Users/christian/Google Drev/Data Science/Bet365/main.py", line 32, in <module>
getScoreH = game.find_element_by_css_selector(scoreH).text
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/selenium/webdriver/remote/webelement.py", line 430, in find_element_by_css_selector
return self.find_element(by=By.CSS_SELECTOR, value=css_selector)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/selenium/webdriver/remote/webelement.py", line 654, in find_element
{"using": by, "value": value})['value']
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/selenium/webdriver/remote/webelement.py", line 628, in _execute
return self._parent.execute(command, params)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/selenium/webdriver/remote/webdriver.py", line 320, in execute
self.error_handler.check_response(response)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/selenium/webdriver/remote/errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.StaleElementReferenceException: Message: stale element reference: element is not attached to the page document
(Session info: chrome=69.0.3497.100)
(Driver info: chromedriver=2.42.591059 (a3d9684d10d61aa0c45f6723b327283be1ebaad8),platform=Mac OS X 10.14.0 x86_64)
The error comes from the last code in my main.py:
# HERE IT BREAKS!:
# Redirects to a games detailed odds page
game.find_element_by_css_selector(oddsBtn).click()
time.sleep(5)
# Go back and keep choose the click the next games details.
obj.find_element(overview).click()
time.sleep(5)
Below is my program. But as I said, the problem comes from the main.py's last few lines of code where I need to go back to an iteration. It seems like it doesn't remember where I left off.
cls_scraper.py:
"""
Class to find element(s) by css selector
"""
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import StaleElementReferenceException
import platform
import time
import os
class Scraper():
def __init__(self, driver):
self.driver = driver
def wait(self, element):
return WebDriverWait(self.driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, element)))
def element_exist_css(self, element):
try:
self.driver.find_element_by_css_selector(element)
except NoSuchElementException:
print('Element doesnt exist')
return False
return True
def element_css(self, element):
try:
time.sleep(2)
return WebDriverWait(self.driver, 10).until(EC.element_to_be_clickable(
(By.CSS_SELECTOR, element)))
except StaleElementReferenceException:
print("XB: StaleElementReferenceException")
except WebDriverException:
print("XB: WebDriverException")
def find_elements(self, element):
time.sleep(2)
return self.driver.find_elements_by_css_selector(element)
def find_element(self, element):
time.sleep(2)
return self.driver.find_element_by_css_selector(element)
str_elements.py:
"""
String library to have relevant css selector elements in one place.
"""
""" BET 365 - Overview-page """
# Enter the page
enterPage = '#TopPromotionMainArea'
# Page with live odds
inPlay = 'body > div:nth-child(1) > div > div:nth-child(1) > div > div.hm-HeaderModule_Primary > div.hm-BigButtons > nav > a:nth-child(2)'
# Element containing relevent games and info about time, score etc, se below.
games = 'div.ipo-FixtureRenderer.ipo-Competition_Container > div'
# For each game in games, these elements can be found:
teamH = 'div.ipo-TeamStack > div:nth-child(1)'
teamA = 'div.ipo-TeamStack > div:nth-child(2)'
scoreH = 'div.ipo-TeamPoints_TeamScore.ipo-TeamPoints_TeamScore-teamone'
scoreA = 'div.ipo-TeamPoints_TeamScore.ipo-TeamPoints_TeamScore-teamtwo'
gameTime = 'div.ipo-InPlayTimer'
# The redirection in order to get all kinds of odds from a match
# Main overview-page only show a part of it.
oddsBtn = 'div.ipo-FixtureEventCountButton_EventCountWrapper'
# The overview tab to see all the live games
overview = 'div.ip-ControlBar > span.ip-ControlBar_ButtonBar > div:nth-child(1)'
# Choose english language
langTab = 'body > div:nth-child(1) > div > div:nth-child(1) > div > div.hm-HeaderModule_Secondary > div.hm-HeaderModule_Menus > div.hm-LanguageDropDownSelections.hm-DropDownSelections > a'
pickEng = 'body > div:nth-child(1) > div > div:nth-child(1) > div > div.hm-HeaderModule_Secondary > div.hm-HeaderModule_Menus > div.hm-LanguageDropDownSelections.hm-DropDownSelections.hm-DropDownSelections_Selected > div > div > a:nth-child(1)'
# Get a better overview
allMarkets = 'body > div:nth-child(1) > div > div.wc-PageView > div.wc-PageView_Main.wc-InPlayPage_MainContainer > div > div > div.ipo-OverViewView > div > div > div > div.ipo-OverViewDetail > div.ipo-OverViewDetail_Container.ipo-Classification > div.ipo-ClassificationHeader_Header.ipo-ClassificationHeader_Header-1.ipo-ClassificationHeader_Header-lightgreenborder.ipo-ClassificationHeader_Header-moremarkets > div.ipo-ClassificationHeader_MarketsButtonOuterWrapper > div > div.ipo-ClassificationHeader_MarketsButton.ipo-ClassificationHeader_MarketsButton-transparent'
""" BET 365 - Odds-page """
# Collect all the odds from the redirection page.
main.py:
""" Run program from here """
from str_elements import *
from cls_scraper import *
from browser.path import *
import time
if __name__ == '__main__':
print("Welcome \n")
# Open website
options = webdriver.ChromeOptions()
driver = webdriver.Chrome(driver_path, chrome_options=options)
driver.get('https://www.bet365.dk/#/HO/')
""" Click relevant elements """
print("Bet365: Pressing buttons ...")
obj = Scraper(driver)
obj.element_css(enterPage).click() # Enters the bet365 main page
obj.element_css(inPlay).click() # Presses the in-play tab
obj.element_css(langTab).click() # Choose languages
obj.element_css(pickEng).click() # Choose english
obj.element_css(overview).click() # Shows all live games
obj.element_css(allMarkets).click() # Better overview
print("Bet365: Collecting game data ...")
# All live games
liveContainer = obj.find_elements(games) # Contains a list of running games
for game in liveContainer:
getScoreH = game.find_element_by_css_selector(scoreH).text
getScoreA = game.find_element_by_css_selector(scoreA).text
getTeamH = game.find_element_by_css_selector(teamH).text
getTeamA = game.find_element_by_css_selector(teamA).text
getTime = game.find_element_by_css_selector(gameTime).text
print("Score: ", getScoreH, "-", getScoreA)
print("GameTime:", getTime)
print("HomeTeam:", getTeamH)
print("AwayTeam:", getTeamA)
print("")
## HERE IT BREAKS!:
## Redirects to a games detailed odds page
# game.find_element_by_css_selector(oddsBtn).click()
# time.sleep(5)
## Go back and keep choose the click the next games details.
# obj.find_element(overview).click()
# time.sleep(5)