We Keep Coding

I want to extract all the urls from number of 10 pages in python using slenium

Here below I mentioned my code and I want to get multiple urls from web page but not from the end the limit of my page is 10. I need only 10 pages urls.
My Code :
`
import requests
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
import time
array = []
driver = webdriver.Chrome()
driver.get('https://google.com')
search = driver.find_element("name","q").send_keys("websites"+Keys.RETURN)
urls = "https://www.google.com/search?q=websites&sxsrf=ALiCzsZmfT1H8dxBOig9KvuRbtnQUtVTtQ%3A1668401426603&source=hp&ei=EslxY8GlIZyM4-EPjqOLoAk&iflsig=AJiK0e8AAAAAY3HXIiUkRUcpwQ84iKLerx9VqGixFmVk&ved=0ahUKEwjB9vzS76z7AhUcxjgGHY7RApQQ4dUDCAk&uact=5&oq=websites&gs_lcp=Cgdnd3Mtd2l6EAMyCAgAEIAEELEDMgsIABCABBCxAxCDATILCAAQgAQQsQMQgwEyBQgAEIAEMgsIABCABBCxAxCDATIFCAAQgAQyBQgAEIAEMgcIABCABBAKMgUIABCABDILCAAQgAQQsQMQgwE6BwgjEOoCECc6BwguEOoCECc6CwguEIMBELEDEIAEOggILhCDARCxAzoICAAQsQMQgwE6BAgjECc6DgguEIAEELEDEMcBENEDOhEILhCABBCxAxCDARDHARCvAVDUBFj1E2D0FWgBcAB4AIAB4QGIAeMJkgEFMC43LjGYAQCgAQGwAQo&sclient=gws-wiz"
driver.get(urls)
elems = driver.find_elements("xpath","/html/body/div/div/div/div/div[2]/div[2]/div/div/div/div/div/div[1]/div/a")
while True:
link = driver.find_elements("xpath","/html/body/div[7]/div/div[11]/div/div[4]/div/div[2]/table/tbody/tr/td[12]/a")
if link == 1 :
print("No more pages left")
break
else:
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.LINK_TEXT, 'Next'))).click()
elems = driver.find_elements("xpath","/html/body/div/div/div/div/div[2]/div[2]/div/div/div/div/div/div[1]/div/a")
for elem in elems:
array = elem.get_attribute("href")
print(array)
`

Selenium error' list' object has no attribute 'text'

from selenium.webdriver import Chrome
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
driver.get('https://www.suwon.go.kr/sw-www/www01/www01-01.jsp?q_tabs=open')
driver.implicitly_wait(3)
content = driver.find_element(By.TAG_NAME, 'iframe')
driver.switch_to.frame(content)
dropdown = Select(driver.find_element(By.XPATH,'//*[#id="dateType"]'))
dropdown.select_by_index(4)
driver.find_element(By.XPATH,'//*[#id="searchBtn"]').click()
complaint_list = []
contents_list = []
def complaint_Scraping():
for i in range(1,78):
titles = driver.find_elements(By.CSS_SELECTOR,'tbody > tr > td.left')
for complaint in titles:
name = BeautifulSoup(complaint.text, "html.parser")
complaint_list.append(name)
a = driver.find_elements(By.CSS_SELECTOR,' tbody > tr > td.left > a')
for content in a:
content.click()
time.sleep(2)
ancient_html = driver.find_elements(By.XPATH,'//*[#id="txt"]/div[1]/div[1]/div/div[2]')
content = BeautifulSoup(ancient_html.text, "html.parser")
contents_list.append(content)
driver.back()
complaint_Scraping()
i dont'know what's wrong in here
I can get all the titles, but It's not working if I try to get contents of the titles. first page may possible, but other things can't, please let me solve the problem.

ancient_html = driver.find_elements(By.XPATH,'//*[#id="txt"]/div[1]/div[1]/div/div[2]')
content = BeautifulSoup(ancient_html.text, "html.parser")
find_elements() returns a list of elements. ancient_html is a list.

How to get text from td tag / Beginner Selenium Python web scraping

I want to create a CSV file with 5 columns. It seems like I can fill the CSV file, but I cannot get the pure text of the underlying table from the website. what am I doing wrong?
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
import pandas as pd
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
from selenium.webdriver.support.ui import Select
PATH = "C:/Users/patri/Desktop/Driver/chromedriver.exe"
driver = webdriver.Chrome(executable_path=PATH)
driver.maximize_window() # For maximizing window
driver.get("https://www.uke.jobs/")
time.sleep(5)
all_data = driver.find_elements(By.XPATH, "//tr[#data-head]")
ref_code = []
stellen_titel = []
fachbereich = []
taetigkeitsbereich = []
unternehmen = []
for a in all_data:
ref_code.append(driver.find_element(By.XPATH, "//tr/td[1]"))
stellen_titel.append(driver.find_element(By.XPATH, "//tr/td[2]"))
fachbereich.append(driver.find_element(By.XPATH, "//tr/td[3]"))
taetigkeitsbereich.append(driver.find_element(By.XPATH, "//tr/td[4]"))
unternehmen.append(driver.find_element(By.XPATH, "//tr/td[5]"))
df = pd.DataFrame({'ref': ref_code, 'titel': stellen_titel, 'fachbereich': fachbereich, 'taetigkeit': taetigkeitsbereich, 'unternehmen': unternehmen})
df.to_csv('UKETEST.csv')
print(df)

how to get product price from different size selenium python

I want to scrape product information from this page . This product have three different size and price will be change if I select different size from drop-down section. Right now my scraper only can scrape default price after first time initially page load which is 35 for 1kg. How I will scrape price for 500g and 250g. here is my code:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
#argument for incognito Chrome
#argument for incognito Chrome
option = Options()
option.add_argument("--incognito")
browser = webdriver.Chrome(options=option)
browser.get("https://boutique.cafebarista.ca/products/cremone?variant=18033418797121")
product_title = browser.find_element_by_xpath('//h1[#class="product-name"]')
long_description = browser.find_element_by_xpath('//div[#class="product-landing-container"]')
price=browser.find_element_by_xpath('//div[#class="product-btn-price ProductPrice"]')
print(product_title.text,long_description.text,price.text)
browser.quit()

With .find_elements_by_css_selector you can get each text without clicking the weight drop down first, this is the selector I mean:
nav[id="w-dropdown-list-16"] > a > div
And you can also click on each of these elements using .execute_script
Try following code:
driver.get('https://boutique.cafebarista.ca/products/autentico?variant=18033459331137')
weight_list = driver.find_elements_by_css_selector('nav[id="w-dropdown-list-16"] > a > div')
for weight in weight_list:
driver.execute_script('arguments[0].click();', weight)
price = driver.find_element_by_id('ProductPrice').text
print(weight.get_attribute('innerHTML') +' ' +price)

Try below solution
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium import webdriver
browser = webdriver.Chrome(executable_path=r"C:\New folder\chromedriver.exe")
browser.maximize_window()
wait = WebDriverWait(browser, 20)
browser.get("https://boutique.cafebarista.ca/products/cremone?variant=18033418797121")
kg_button=browser.find_element_by_xpath("//div[#id='w-dropdown-toggle-16']")
kg_button.click()
list =wait.until(EC.presence_of_all_elements_located((By.XPATH, "//nav[#id='w-dropdown-list-16']//a")))
kg_button.click()
for element in list:
kg_button.click()
actionChains = ActionChains(browser)
actionChains.move_to_element(element).click().perform()
price = browser.find_element_by_xpath("//div[#id='ProductPrice']")
print product_title.text
print element.text
print price.text
browser.quit()

How do I extract a list of URLs off a page with selenium?

I am attempting to extract all urls that https://shop.freedommobile.ca/devices has when you click the 'see options' button under each phone and place them into a list of strings.
I am using python with Selenium and wait libraries.
Ive already tried using .text in my parameters. However, I keep running into an error that states:
typeError: 'str' object is not callable
line 17 is the issue.
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
driver = webdriver.Chrome()
class phoneCost:
driver.get("https://shop.freedommobile.ca/devices")
# extract the names of the phones
wait = WebDriverWait(driver, 20) #10 second wait
XPathLocation = """B//*[#id="skip-navigation"]/div/div/div[1]/div/div[2]/a'"""
phonePlanLinksRaw = wait.until(EC.presence_of_all_elements_located(By.XPATH(XPathLocation)))
phonePlanLinks = []
for element in range(len(phonePlanLinksRaw)):
link = element
phonePlanLinks.append(str(link))
numLink = 1
for element in range(len(phonePlanLinks)):
print("phone " + str(numLink) + " : " + phonePlanLinks[element])
numLink += 1
should return a list of urls in string format:
[https://shop.freedommobile.ca/devices/Apple/iPhone_XS_Max?sku=190198786135&planSku=Freedom%20Big%20Gig%2015GB
,
https://shop.freedommobile.ca/devices/Apple/iPhone_XS?sku=190198790569&planSku=Freedom%20Big%20Gig%
,
https://shop.freedommobile.ca/devices/Apple/iPhone_XR?sku=190198776631&planSku=Freedom%20Big%20Gig%2015GB]
Any help is appreciated
Thank you

Here is the logic that you should use.
WebDriverWait(driver,10).until(EC.presence_of_all_elements_located((By.XPATH,"//div[starts-with(#class,'deviceListItem')]/a")))
mblOptions = driver.find_elements_by_xpath("//div[starts-with(#class,'deviceListItem')]/a")
mblUrls = []
for mblOption in mblOptions:
mblUrls.append(mblOption.get_attribute('href'))
print (mblUrls)
output:
['https://shop.freedommobile.ca/devices/Apple/iPhone_XS_Max?sku=190198786135&planSku=Freedom%20Big%20Gig%2015GB', 'https://shop.freedommobile.ca/devices/Apple/iPhone_XS?sku=190198790569&planSku=Freedom%20Big%20Gig%2015GB', 'https://shop.freedommobile.ca/devices/Apple/iPhone_XR?sku=190198776631&planSku=Freedom%20Big%20Gig%2015GB', 'https://shop.freedommobile.ca/devices/Apple/iPhone_8_Plus?sku=190198454249&planSku=Freedom%20Big%20Gig%20%2B%20Talk%205GB', 'https://shop.freedommobile.ca/devices/Apple/iPhone_8?sku=190198450944&planSku=Freedom%20Big%20Gig%20%2B%20Talk%205GB', 'https://shop.freedommobile.ca/devices/Samsung/Galaxy_S10+?sku=887276301570&planSku=Freedom%20Big%20Gig%20%2B%20Talk%205GB', 'https://shop.freedommobile.ca/devices/Samsung/Galaxy_S10?sku=887276312163&planSku=Freedom%20Big%20Gig%20%2B%20Talk%2015GB', 'https://shop.freedommobile.ca/devices/Samsung/Galaxy_S10e?sku=887276313870&planSku=Freedom%20Big%20Gig%2015GB', 'https://shop.freedommobile.ca/devices/Samsung/Galaxy_Tab_A_8_LTE?sku=887276299440&planSku=Promo%20Tablet%2015', 'https://shop.freedommobile.ca/devices/Samsung/Galaxy_Note9?sku=887276279916&planSku=Freedom%20Big%20Gig%2015GB', 'https://shop.freedommobile.ca/devices/Samsung/Galaxy_S9?sku=887276250861&planSku=Freedom%20Big%20Gig%20%2B%20Talk%205GB', 'https://shop.freedommobile.ca/devices/Motorola/G7_Power?sku=723755134249&planSku=Freedom%20Big%20Gig%20%2B%20Talk%205GB', 'https://shop.freedommobile.ca/devices/Motorola/Moto_E5_Play?sku=723755125940&planSku=Freedom%20LTE%2B3G%209.5GB%20Promo', 'https://shop.freedommobile.ca/devices/Google/Pixel_3a?sku=842776111326&planSku=Freedom%20Big%20Gig%20%2B%20Talk%205GB', 'https://shop.freedommobile.ca/devices/Google/Pixel_3?sku=842776109798&planSku=Freedom%20Big%20Gig%20%2B%20Talk%2010GB', 'https://shop.freedommobile.ca/devices/Google/Pixel_3_XL?sku=842776109828&planSku=Freedom%20Big%20Gig%20%2B%20Talk%2010GB', 'https://shop.freedommobile.ca/devices/ZTE/Z557?sku=885913107448&planSku=Freedom%20500MB', 'https://shop.freedommobile.ca/devices/LG/G7_ThinQ?sku=652810830737&planSku=Freedom%20Big%20Gig%20%2B%20Talk%205GB', 'https://shop.freedommobile.ca/devices/Huawei/P30_lite?sku=886598061131&planSku=Freedom%20Big%20Gig%20%2B%20Talk%205GB', 'https://shop.freedommobile.ca/devices/Huawei/Mate_20_Pro?sku=886598058964&planSku=Freedom%20Big%20Gig%20%2B%20Talk%2010GB', 'https://shop.freedommobile.ca/devices/LG/X_Power_3?sku=652810831130&planSku=Freedom%20LTE%2B3G%209.5GB%20Promo', 'https://shop.freedommobile.ca/devices/LG/G8_ThinQ?sku=652810832434&planSku=Freedom%20Big%20Gig%20%2B%20Talk%2010GB', 'https://shop.freedommobile.ca/devices/LG/Q_Stylo_+?sku=652810831222&planSku=Freedom%202GB', 'https://shop.freedommobile.ca/devices/Alcatel/GoFLIP?sku=889063504010&planSku=Freedom%20500MB', 'https://shop.freedommobile.ca/devices/Bring_Your/Own_Device?sku=byod']

Try using list comprehension to achieve the reults. Just take a look at this portion (By.XPATH(XPathLocation))) that you used which should be wait.until(EC.visibility_of_all_elements_located((By.XPATH, "some_xpath"))).
Rectified one is more like:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
with webdriver.Chrome() as driver:
wait = WebDriverWait(driver, 10)
driver.get("https://shop.freedommobile.ca/devices")
item_links = [item.get_attribute("href") for item in wait.until(EC.visibility_of_all_elements_located((By.XPATH, "//a[contains(#class,'__DeviceDetailsButton')]")))]
print(item_links)

To extract all urls that https://shop.freedommobile.ca/devices has using Selenium you have to induce WebDriverWait for the visibility_of_all_elements_located() and you can use the following Locator Strategy:
Code Block:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
options = webdriver.ChromeOptions()
options.add_argument("start-maximized")
# options.add_argument('disable-infobars')
driver = webdriver.Chrome(chrome_options=options, executable_path=r'C:\WebDrivers\chromedriver.exe')
driver.get("https://shop.freedommobile.ca/devices")
print([my_elem.get_attribute("href") for my_elem in WebDriverWait(driver, 20).until(EC.visibility_of_all_elements_located((By.XPATH, "//a[text()='See Options']")))])
Console Output:
['https://shop.freedommobile.ca/devices/Apple/iPhone_XS_Max?sku=190198786135&planSku=Freedom%20Big%20Gig%2015GB', 'https://shop.freedommobile.ca/devices/Apple/iPhone_XS?sku=190198790569&planSku=Freedom%20Big%20Gig%2015GB', 'https://shop.freedommobile.ca/devices/Apple/iPhone_XR?sku=190198776631&planSku=Freedom%20Big%20Gig%2015GB', 'https://shop.freedommobile.ca/devices/Apple/iPhone_8_Plus?sku=190198454249&planSku=Freedom%20Big%20Gig%20%2B%20Talk%205GB', 'https://shop.freedommobile.ca/devices/Apple/iPhone_8?sku=190198450944&planSku=Freedom%20Big%20Gig%20%2B%20Talk%205GB', 'https://shop.freedommobile.ca/devices/Samsung/Galaxy_S10+?sku=887276301570&planSku=Freedom%20Big%20Gig%20%2B%20Talk%205GB', 'https://shop.freedommobile.ca/devices/Samsung/Galaxy_S10?sku=887276312163&planSku=Freedom%20Big%20Gig%20%2B%20Talk%2015GB', 'https://shop.freedommobile.ca/devices/Samsung/Galaxy_S10e?sku=887276313870&planSku=Freedom%20Big%20Gig%2015GB', 'https://shop.freedommobile.ca/devices/Samsung/Galaxy_Tab_A_8_LTE?sku=887276299440&planSku=Promo%20Tablet%2015', 'https://shop.freedommobile.ca/devices/Samsung/Galaxy_Note9?sku=887276279916&planSku=Freedom%20Big%20Gig%2015GB', 'https://shop.freedommobile.ca/devices/Samsung/Galaxy_S9?sku=887276250861&planSku=Freedom%20Big%20Gig%20%2B%20Talk%205GB', 'https://shop.freedommobile.ca/devices/Motorola/G7_Power?sku=723755134249&planSku=Freedom%20Big%20Gig%20%2B%20Talk%205GB', 'https://shop.freedommobile.ca/devices/Motorola/Moto_E5_Play?sku=723755125940&planSku=Freedom%20LTE%2B3G%209.5GB%20Promo', 'https://shop.freedommobile.ca/devices/Google/Pixel_3a?sku=842776111326&planSku=Freedom%20Big%20Gig%20%2B%20Talk%205GB', 'https://shop.freedommobile.ca/devices/Google/Pixel_3?sku=842776109798&planSku=Freedom%20Big%20Gig%20%2B%20Talk%2010GB', 'https://shop.freedommobile.ca/devices/Google/Pixel_3_XL?sku=842776109828&planSku=Freedom%20Big%20Gig%20%2B%20Talk%2010GB', 'https://shop.freedommobile.ca/devices/ZTE/Z557?sku=885913107448&planSku=Freedom%20500MB', 'https://shop.freedommobile.ca/devices/LG/G7_ThinQ?sku=652810830737&planSku=Freedom%20Big%20Gig%20%2B%20Talk%205GB', 'https://shop.freedommobile.ca/devices/Huawei/P30_lite?sku=886598061131&planSku=Freedom%20Big%20Gig%20%2B%20Talk%205GB', 'https://shop.freedommobile.ca/devices/Huawei/Mate_20_Pro?sku=886598058964&planSku=Freedom%20Big%20Gig%20%2B%20Talk%2010GB', 'https://shop.freedommobile.ca/devices/LG/X_Power_3?sku=652810831130&planSku=Freedom%20LTE%2B3G%209.5GB%20Promo', 'https://shop.freedommobile.ca/devices/LG/G8_ThinQ?sku=652810832434&planSku=Freedom%20Big%20Gig%20%2B%20Talk%2010GB', 'https://shop.freedommobile.ca/devices/LG/Q_Stylo_+?sku=652810831222&planSku=Freedom%202GB', 'https://shop.freedommobile.ca/devices/Alcatel/GoFLIP?sku=889063504010&planSku=Freedom%20500MB', 'https://shop.freedommobile.ca/devices/Bring_Your/Own_Device?sku=byod']