I want to get and print all the Urls of the products in one of the AliExpress pages, it worked but just the first 12 urls that show and not all the urls on the page, normally it should be 60 urls because every page in AliExpress contains 60 products. How can I extract all the 60 urls. Any help is highly appreciated.
This is my code :
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.by import By
from time import sleep
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
driver.get("https://www.aliexpress.com/wholesale?trafficChannel=main&d=y&CatId=0&SearchText=smart+lock<ype=wholesale&SortType=total_tranpro_desc")
urls = [a.get_attribute('href') for a in driver.find_elements(By.CSS_SELECTOR,'a[href*="/item/"]')]
for url in urls:
print(url)
The web page there not presenting all the products. It uses specific loading strategy to present only currently visible products.
To grab all the products you will have to scroll the page collecting the presented products during the scrolling. The URLs should be stored in set to avoid duplicates.
I tried the following simple solution and it worked for me:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
import time
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(service=webdriver_service, options=options)
url = "https://www.aliexpress.com/wholesale?trafficChannel=main&d=y&CatId=0&SearchText=smart+lock<ype=wholesale&SortType=total_tranpro_desc"
driver.get(url)
urls = set()
for i in range(5):
block = [a.get_attribute('href') for a in driver.find_elements(By.CSS_SELECTOR, 'a[href*="/item/"]')]
urls.update(block)
driver.execute_script("window.scrollBy(0, arguments[0]);", 600)
time.sleep(2)
for url in urls:
print(url)
The output is:
https://he.aliexpress.com/item/1005004505011076.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-33&pdp_ext_f=%7B%22sku_id%22%3A%2212000029397314035%22%7D&pdp_npi=2%40dis%21ILS%21680.47%21238.16%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000029397314035%21sea&curPageLogUid=eOPom12i1YdL
https://he.aliexpress.com/item/1005003118129589.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-3&pdp_ext_f=%7B%22sku_id%22%3A%2212000024195825216%22%7D&pdp_npi=2%40dis%21ILS%2117.75%2114.38%21%21%210.0%21%21%400b0a0ac216625452616837499e4d53%2112000024195825216%21sea&curPageLogUid=b4KiLx28AEal
https://he.aliexpress.com/item/1005004314575646.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-27&pdp_ext_f=%7B%22sku_id%22%3A%2212000028723076890%22%7D&pdp_npi=2%40dis%21ILS%21626.53%21307.0%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000028723076890%21sea&curPageLogUid=iOoGBaB9fk1i
https://he.aliexpress.com/item/4000154297870.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-4&pdp_ext_f=%7B%22sku_id%22%3A%2212000017734780587%22%7D&pdp_npi=2%40dis%21ILS%2176.86%2139.18%21%21%217.4%21%21%400b0a0ac216625452616837499e4d53%2112000017734780587%21sea&curPageLogUid=iMbrWCxxlNRK
https://he.aliexpress.com/item/1005002868522660.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-28&pdp_ext_f=%7B%22sku_id%22%3A%2212000022534558777%22%7D&pdp_npi=2%40dis%21ILS%2163.46%2138.69%21%21%2121.15%21%21%400b0a0ac216625452616837499e4d53%2112000022534558777%21sea&curPageLogUid=kzYKVU82QxVZ
https://he.aliexpress.com/item/1005004495398206.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-44&pdp_ext_f=%7B%22sku_id%22%3A%2212000029361338239%22%7D&pdp_npi=2%40dis%21ILS%21303.25%21239.56%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000029361338239%21sea&curPageLogUid=ZCNQVDFjiW24
https://he.aliexpress.com/item/1005002843575207.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-30&pdp_ext_f=%7B%22sku_id%22%3A%2212000026607998519%22%7D&pdp_npi=2%40dis%21ILS%21294.41%21232.57%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000026607998519%21sea&curPageLogUid=dKGPq84ciu6A
https://he.aliexpress.com/item/4001166182060.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-14&pdp_ext_f=%7B%22sku_id%22%3A%2210000014983376216%22%7D&pdp_npi=2%40dis%21ILS%2183.98%2167.17%21%21%213.51%21%21%400b0a0ac216625452616837499e4d53%2110000014983376216%21sea&curPageLogUid=5P64jj4BsxMb
https://he.aliexpress.com/item/1005002864846136.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-31&pdp_ext_f=%7B%22sku_id%22%3A%2212000027311675115%22%7D&pdp_npi=2%40dis%21ILS%21321.74%21254.18%21%21%2130.24%21%21%400b0a0ac216625452616837499e4d53%2112000027311675115%21sea&curPageLogUid=wIquG44FDWvS
https://he.aliexpress.com/item/1005004407267255.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-25&pdp_ext_f=%7B%22sku_id%22%3A%2212000029260939933%22%7D&pdp_npi=2%40dis%21ILS%211117.23%21893.79%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000029260939933%21sea&curPageLogUid=L9QVMI7Dkc9L
https://he.aliexpress.com/item/1005003202983418.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-13&pdp_ext_f=%7B%22sku_id%22%3A%2212000024642176626%22%7D&pdp_npi=2%40dis%21ILS%2176.75%2127.75%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000024642176626%21sea&curPageLogUid=w934YlWjZpSk
https://he.aliexpress.com/item/10000345027263.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-40&pdp_ext_f=%7B%22sku_id%22%3A%2220000000178547076%22%7D&pdp_npi=2%40dis%21ILS%2173.66%2147.88%21%21%216.31%21%21%400b0a0ac216625452616837499e4d53%2120000000178547076%21sea&curPageLogUid=1CXW2eZmYyoz
https://he.aliexpress.com/item/1005003782715703.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-41&pdp_ext_f=%7B%22sku_id%22%3A%2212000028520788363%22%7D&pdp_npi=2%40dis%21ILS%21733.13%21403.22%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000028520788363%21sea&curPageLogUid=DzR0GUWN6DqQ
https://he.aliexpress.com/item/1005003093642821.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-11&pdp_ext_f=%7B%22sku_id%22%3A%2212000025801875061%22%7D&pdp_npi=2%40dis%21ILS%21823.63%21271.78%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000025801875061%21sea&curPageLogUid=yVO5hwgoHmV1
https://he.aliexpress.com/item/4000918150378.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-23&pdp_ext_f=%7B%22sku_id%22%3A%2212000021385410206%22%7D&pdp_npi=2%40dis%21ILS%21178.69%21146.52%21%21%2186.4%21%21%400b0a0ac216625452616837499e4d53%2112000021385410206%21sea&curPageLogUid=C6dkvTMogJ62
https://he.aliexpress.com/item/1005001900549701.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-16&pdp_ext_f=%7B%22sku_id%22%3A%2212000024274381461%22%7D&pdp_npi=2%40dis%21ILS%21188.58%21105.62%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000024274381461%21sea&curPageLogUid=d1dAHgBFOsqF
https://he.aliexpress.com/item/1005003257440167.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-29&pdp_ext_f=%7B%22sku_id%22%3A%2212000029883679176%22%7D&pdp_npi=2%40dis%21ILS%21891.4%21410.06%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000029883679176%21sea&curPageLogUid=D3UOiF7Eswwj
https://he.aliexpress.com/item/33023975249.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-19&pdp_ext_f=%7B%22sku_id%22%3A%2212000026754718473%22%7D&pdp_npi=2%40dis%21ILS%21132.1%2147.57%21%21%2112.0%21%21%400b0a0ac216625452616837499e4d53%2112000026754718473%21sea&curPageLogUid=iVmMqv3Hv2dw
https://he.aliexpress.com/item/1005004105323148.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-35&pdp_ext_f=%7B%22sku_id%22%3A%2212000028159854219%22%7D&pdp_npi=2%40dis%21ILS%21497.87%21388.34%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000028159854219%21sea&curPageLogUid=ScGffoZSAzHu
https://he.aliexpress.com/item/1005003076385676.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-38&pdp_ext_f=%7B%22sku_id%22%3A%2212000023907005619%22%7D&pdp_npi=2%40dis%21ILS%2186.71%2124.94%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000023907005619%21sea&curPageLogUid=QArTgWyf3MS7
https://he.aliexpress.com/item/1005003731084162.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-2&pdp_ext_f=%7B%22sku_id%22%3A%2212000029315337088%22%7D&pdp_npi=2%40dis%21ILS%21743.72%21327.24%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000029315337088%21sea&curPageLogUid=aMsEw0Z36uSN
https://he.aliexpress.com/item/4001010240435.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-43&pdp_ext_f=%7B%22sku_id%22%3A%2210000013511866229%22%7D&pdp_npi=2%40dis%21ILS%2129.36%213.02%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2110000013511866229%21sea&curPageLogUid=z1vD7tD9W5MZ
https://he.aliexpress.com/item/4000154478484.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-8&pdp_ext_f=%7B%22sku_id%22%3A%2212000021350825852%22%7D&pdp_npi=2%40dis%21ILS%2178.64%2135.39%21%21%217.4%21%21%400b0a0ac216625452616837499e4d53%2112000021350825852%21sea&curPageLogUid=yRWy6gHc4b1E
https://he.aliexpress.com/item/1005003346086758.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-32&pdp_ext_f=%7B%22sku_id%22%3A%2212000025331814334%22%7D&pdp_npi=2%40dis%21ILS%2195.31%2158.12%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000025331814334%21sea&curPageLogUid=iGvJbXPDD3cg
https://he.aliexpress.com/item/4000065467682.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-6&pdp_ext_f=%7B%22sku_id%22%3A%2210000015061976915%22%7D&pdp_npi=2%40dis%21ILS%21281.64%21191.53%21%21%21162.48%21%21%400b0a0ac216625452616837499e4d53%2110000015061976915%21sea&curPageLogUid=1qiD9mBYf6KM
https://he.aliexpress.com/item/1005002676337744.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-1&pdp_ext_f=%7B%22sku_id%22%3A%2212000021669614390%22%7D&pdp_npi=2%40dis%21ILS%21561.28%21392.91%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000021669614390%21sea&curPageLogUid=Y1Mvi9ey1Hwl
https://he.aliexpress.com/item/4000834334480.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-34&pdp_ext_f=%7B%22sku_id%22%3A%2212000028610539908%22%7D&pdp_npi=2%40dis%21ILS%21162.83%21138.42%21%21%2168.96%21%21%400b0a0ac216625452616837499e4d53%2112000028610539908%21sea&curPageLogUid=6CijCm642BQh
https://he.aliexpress.com/item/4001031327478.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-7&pdp_ext_f=%7B%22sku_id%22%3A%2212000015965032899%22%7D&pdp_npi=2%40dis%21ILS%211037.26%21383.79%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000015965032899%21sea&curPageLogUid=fLjjoL5HUrxi
https://he.aliexpress.com/item/1005004262179786.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-12&pdp_ext_f=%7B%22sku_id%22%3A%2212000028573420101%22%7D&pdp_npi=2%40dis%21ILS%21679.22%21319.24%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000028573420101%21sea&curPageLogUid=n4io2MIKAEsP
https://he.aliexpress.com/item/1005003515525914.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-0&pdp_ext_f=%7B%22sku_id%22%3A%2212000026134836061%22%7D&pdp_npi=2%40dis%21ILS%21493.37%21360.15%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000026134836061%21sea&curPageLogUid=qY8tyv2zwcdK
https://he.aliexpress.com/item/1005003230965691.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-26&pdp_ext_f=%7B%22sku_id%22%3A%2212000024764885745%22%7D&pdp_npi=2%40dis%21ILS%21188.3%21133.68%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000024764885745%21sea&curPageLogUid=CJHjTgOCDRJ3
https://he.aliexpress.com/item/4000975407287.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-10&pdp_ext_f=%7B%22sku_id%22%3A%2210000015021478939%22%7D&pdp_npi=2%40dis%21ILS%21680.62%21299.47%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2110000015021478939%21sea&curPageLogUid=WDdJT8cOU94O
https://he.aliexpress.com/item/1005003187314155.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-42&pdp_ext_f=%7B%22sku_id%22%3A%2212000024567051659%22%7D&pdp_npi=2%40dis%21ILS%21157.54%2194.54%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000024567051659%21sea&curPageLogUid=4qQvpAeygSFw
https://he.aliexpress.com/item/32954893736.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-39&pdp_ext_f=%7B%22sku_id%22%3A%2210000002371991125%22%7D&pdp_npi=2%40dis%21ILS%21303.58%21100.18%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2110000002371991125%21sea&curPageLogUid=yDxMTsGMA1m1
https://he.aliexpress.com/item/1005002548126691.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-15&pdp_ext_f=%7B%22sku_id%22%3A%2212000021089170892%22%7D&pdp_npi=2%40dis%21ILS%2171.49%2118.2%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000021089170892%21sea&curPageLogUid=xfzmTqNWeUEV
https://he.aliexpress.com/item/4000735115535.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-17&pdp_ext_f=%7B%22sku_id%22%3A%2212000027051383811%22%7D&pdp_npi=2%40dis%21ILS%21223.41%21176.51%21%21%2128.2%21%21%400b0a0ac216625452616837499e4d53%2112000027051383811%21sea&curPageLogUid=FasdXFlpMCuE
https://he.aliexpress.com/item/1005002659060326.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-20&pdp_ext_f=%7B%22sku_id%22%3A%2212000021594037009%22%7D&pdp_npi=2%40dis%21ILS%211144.03%21800.83%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000021594037009%21sea&curPageLogUid=qWxA5NkkHX7u
https://he.aliexpress.com/item/1005003050882550.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-21&pdp_ext_f=%7B%22sku_id%22%3A%2212000023509149328%22%7D&pdp_npi=2%40dis%21ILS%2178.01%2135.5%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000023509149328%21sea&curPageLogUid=eazJyQ2xYlTa
https://he.aliexpress.com/item/1005001429138105.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-5&pdp_ext_f=%7B%22sku_id%22%3A%2212000016106374836%22%7D&pdp_npi=2%40dis%21ILS%21908.29%21272.49%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000016106374836%21sea&curPageLogUid=UTqfAcwKjxt3
https://he.aliexpress.com/item/1005003575980819.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-9&pdp_ext_f=%7B%22sku_id%22%3A%2212000026346773711%22%7D&pdp_npi=2%40dis%21ILS%2156.48%2137.29%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000026346773711%21sea&curPageLogUid=t6N9K9ie8MII
https://he.aliexpress.com/item/1005004406234563.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-24&pdp_ext_f=%7B%22sku_id%22%3A%2212000029068345270%22%7D&pdp_npi=2%40dis%21ILS%21334.62%21167.31%21%21%2119.93%21%21%400b0a0ac216625452616837499e4d53%2112000029068345270%21sea&curPageLogUid=ERz9XojcigBs
https://he.aliexpress.com/item/1005003208283080.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-18&pdp_ext_f=%7B%22sku_id%22%3A%2212000024667248681%22%7D&pdp_npi=2%40dis%21ILS%21164.38%21100.25%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000024667248681%21sea&curPageLogUid=Z5qyCqcRmY48
https://he.aliexpress.com/item/1005002983524272.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-37&pdp_ext_f=%7B%22sku_id%22%3A%2212000023060725911%22%7D&pdp_npi=2%40dis%21ILS%21164.45%21146.35%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000023060725911%21sea&curPageLogUid=LPz7kSEHNr3P
https://he.aliexpress.com/item/32964505971.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-22&pdp_ext_f=%7B%22sku_id%22%3A%2212000020850078768%22%7D&pdp_npi=2%40dis%21ILS%21861.0%21266.91%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000020850078768%21sea&curPageLogUid=ehbKLbbMGsal
https://he.aliexpress.com/item/1005004243027957.html?algo_pvid=cf9a5cc9-df53-404b-a8ed-db8a227069e6&algo_exp_id=cf9a5cc9-df53-404b-a8ed-db8a227069e6-36&pdp_ext_f=%7B%22sku_id%22%3A%2212000029030797848%22%7D&pdp_npi=2%40dis%21ILS%21723.76%21325.69%21%21%21%21%21%400b0a0ac216625452616837499e4d53%2112000029030797848%21sea&curPageLogUid=SAk2CRsHhKDu
Process finished with exit code 0
Related
I am trying to scrape a website to get the heading and summary of the news. The problem I am facing is that when we first open the website, a redirect appears and we have to wait 8 seconds for the website to load. The problem I am facing is that the web data that is beign stored is that of the redirect instead of the main website.
from selenium import webdriver
import time
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from bs4 import BeautifulSoup
# Specify the path to the ChromeDriver executable
chrome_driver_path = "C:/webdrivers/chromedriver"
# Initialize the webdriver
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
# Navigate to website
driver.get("https://economictimes.indiatimes.com/markets/stocks/news")
time.sleep(10)
data2, data4 = [], []
while True:
# Extract data
soup = BeautifulSoup(driver.page_source, 'html.parser')
data = soup.find_all("div", {"class": "example-class"})
for item in data:
data2.append(item.find_all('h3'))
data4.append(item.find_all('p'))
try:
# Find the "Load More" button
load_more_button = driver.find_element_by_css_selector("div.autoload_continue")
# Click the button
load_more_button.click()
except:
break
# Close the browser
driver.quit()
print(data2)
You could check for switch to your final url:
wait.until(EC.url_to_be('https://economictimes.indiatimes.com/markets/stocks/news'))
Example
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from webdriver_manager.chrome import ChromeDriverManager
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
url = 'https://economictimes.indiatimes.com/markets/stocks/news'
wait = WebDriverWait(driver, 10)
driver.get(url)
wait.until(EC.url_to_be('https://economictimes.indiatimes.com/markets/stocks/news'))
An ideal approach would be to wait for the News heading within the webpage to be visibible.
Solution
To wait for the News heading to be visibible you need to induce WebDriverWait for the visibility_of_element_located() and you can use either of the following locator strategies:
Using CSS_SELECTOR:
driver.get('https://economictimes.indiatimes.com/markets/stocks/news')
WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.CSS_SELECTOR, "h1.h1")))
Using XPATH:
driver.get('https://economictimes.indiatimes.com/markets/stocks/news')
WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.XPATH, "//h1[#class='h1' and text()='News']")))
Note : You have to add the following imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
Alternative
You can also wait for the Page Title of the webpage to contain Stocks in News Today as follows:
driver.get('https://economictimes.indiatimes.com/markets/stocks/news')
WebDriverWait(driver, 10).until(EC.title_contains("Stocks in News Today"))
References
You can find a couple of relevant detailed discussions in:
Python selenium get page title
How to make selenium wait before getting contents from the actual website which loads after the landing page through IEDriverServer and IE
I am trying to scrape Google results to get URLs of Linkedin accounts:
import variables
import csv
from time import sleep
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.keys import Keys
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from parsel import Selector
from webdriver_manager.chrome import ChromeDriverManager
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
driver.get('https://www.google.com/')
search_bar = driver.find_element(By.NAME, 'q')
sleep(3)
search_bar.send_keys(variables.query)
sleep(3)
search_bar.send_keys(Keys.ENTER)
linkedin_users_urls_list = driver.find_elements(By.XPATH, '//div[#class="yuRUbf"]/a[#href]')
profile_urls = []
// only 11 results
# get urls
[profile_urls.append(users.get_attribute("href")) for users in linkedin_users_urls_list]
print(profile_urls)
// only 11 results
This code works but for some reason I only get like 11 results inprofile_urls array.
Why is this happening if Google returns like several thousand results containing the required URL? I also have changed the Google settings to output maximum number of results per page.
Is there some kind of limitation with XPATH argument?
I am trying to get href but they give me nothing these is page link https://www.nascc.aisc.org/trade-show
import time
from selenium import webdriver
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
driver = webdriver.Chrome("C:\Program Files (x86)\chromedriver.exe")
URL = 'https://www.nascc.aisc.org/trade-show'
driver.get(URL)
page_links =[element.get_attribute('href') for element in
driver.find_elements(By.XPATH, "//table[#class='ffTableSet table table-striped']//a[starts-with(#href, 'https://n2a.goexposoftware.com/events/nascc23/goExpo/exhibitor')]")]
print(page_links)
Here's how to avoid dealing with iframes on that page: (Go directly to the inner site)
from selenium import webdriver
driver = webdriver.Chrome()
URL = 'https://n2a.goexposoftware.com/events/nascc23/goExpo/public/listExhibitorsFrame.php'
driver.get(URL)
page_links = [element.get_attribute("href") for element in
driver.find_elements("css selector",
'[href*="nascc23/goExpo/exhibitor"]')]
print(page_links)
driver.quit()
The element is inside nested iframe you need to switch to both frames.
URL = 'https://www.nascc.aisc.org/trade-show'
driver.get(URL)
WebDriverWait(driver,10).until(EC.frame_to_be_available_and_switch_to_it((By.CSS_SELECTOR,"iframe[name='htmlComp-iframe']")))
WebDriverWait(driver,10).until(EC.frame_to_be_available_and_switch_to_it((By.CSS_SELECTOR,"iframe#geFrame1")))
time.sleep(5)
page_links =[element.get_attribute('href') for element in
driver.find_elements(By.XPATH, "//table[#class='ffTableSet table table-striped']//a[starts-with(#href, 'https://n2a.goexposoftware.com/events/nascc23/goExpo/exhibitor')]")]
print(page_links)
You need to import below libraries
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
import time
from scrapy_selenium import SeleniumRequest
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.select import Select
from selenium import webdriver
url='https://www.aeafa.es/asociados.php?provinput='
driver =webdriver.Chrome('C:\Program Files (x86)\chromedriver.exe')
driver.get(url)
wait = WebDriverWait(driver, 30)
detail=wait.until(EC.element_to_be_clickable((By.XPATH, "//tbody//td[6]")))
detail.click()
Error will be in these:
detail=wait.until(EC.element_to_be_clickable((By.XPATH, "//tbody//td[6]")))
detail.click()
I want that they click on the title of these is page link
and then there will scrape these information. How to scrape these data.
You cannot locate HTML attribute to click on it. Try to replace
detail=wait.until(EC.element_to_be_clickable((By.XPATH, "//td//img//#src"))).click()
with
detail = wait.until(EC.element_to_be_clickable((By.XPATH, "//a[#title='info']")))
detail.click()
UPDATE
Since your ScrapySelenium approach doeasn't seem to work, try common Selenium approach. Then you could adapt it to Scrapy
from selenium import webdriver
driver = webdriver.Chrome()
url = 'https://www.aeafa.es/asociados.php?provinput='
driver.get(url)
for mail in driver.find_elements("xpath", "//p/a[starts-with(#href, 'mailto')]"):
print(mail.get_attribute('textContent'))
Note that you don't need to open each details popup to get required email text
I am scraping one of the URLs like this which loads data via Ajax. When using Firefox it's able to scrape HTML but upon using PhantomJS it return:
<html><head></head><body></body></html>
Code is below:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import selenium.webdriver.support.ui as ui
import sys
import os
from time import sleep
driver = None
url = 'https://sports.bovada.lv/live-betting/event/2391243'
driver = webdriver.PhantomJS('/Setups/phantomjs-1.9.8-macosx/bin/phantomjs')
driver.set_window_size(1128, 768) # optional
driver.get(url)
wait = ui.WebDriverWait(driver, 3000)
sleep(40)
#wait.until(EC.staleness_of(driver.find_element_by_id("coupon")), 'visible')
html = driver.page_source
#userElement = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.ID, "coupon")))
print(html)
Update
Ok this is happening with every URL regardless of ajax or non-Ajax