BeautifulSoup can't find element class in HTML

BeautifulSoup can't find element class in HTML - python

I'm trying to scrape this page have 10 class='name main-name', like this:sample source
but when i code:
import requests
from bs4 import BeautifulSoup
result = requests.get("https://genvita.vn/thu-thach/7-ngay-detox-da-dep-dang-thon-nguoi-khoe-qua-soc-len-den-8-trieu-dong")
c = result.text
soup = BeautifulSoup(c, "html.parser")
comment_items = soup.find_all('div', class_="name main-name")
print(len(comment_items)
but return : 0 not return : 10. I have tried search and use many solutions in stackoverflow but cann't fix

Because div name main-name doens't appear in your DOM . In this case using Selenium is more powerful than BeautifulSoap
from selenium import webdriver
driver_path = r'Your Chrome driver path'
browser = webdriver.Chrome(executable_path=driver_path)
browser.get("https://genvita.vn/thu-thach/7-ngay-detox-da-dep-dang-thon-nguoi-khoe-qua-soc-len-den-8-trieu-dong")
get_element = browser.find_elements_by_css_selector("div[class='name main-name']")
print len(get_element)
browser.close()
OUTPUT :
10
And you can also get names like:
for users in get_element:
print(users.text)
OUTPUT :
Phạm Thị Kim Chi
My Linh Nguyen
Mr Vinh Bảo Hiểm Sức Khoẻ Sắc Đẹp
Ngô Thị Tuyết
Huỳnh Thị Bích Trâm
Linh Trúc Diêm
Nguyen Tu
Nguyen Thom
Hồ Thu Trang
Trầnthịtrắng

As I stated in the comments, it's generated dynamically. So here's an implementation with Selenium:
from selenium import webdriver
from bs4 import BeautifulSoup
url = "https://genvita.vn/thu-thach/7-ngay-detox-da-dep-dang-thon-nguoi-khoe-qua-soc-len-den-8-trieu-dong"
driver = webdriver.Chrome('C:/chromedriver_win32/chromedriver.exe')
driver.get(url)
c = driver.page_source
soup = BeautifulSoup(c, "html.parser")
comment_items = soup.find_all('div', {'class':"name main-name"})
print (len(comment_items))
driver.close()
Output:
print (len(comment_items))
10

You can use beautifulsoup4 select function
import requests
from bs4 import BeautifulSoup
result = requests.get("https://genvita.vn/thu-thach/7-ngay-detox-da-dep-dang-thon-nguoi-khoe-qua-soc-len-den-8-trieu-dong")
c = result.text
soup = BeautifulSoup(c, "html.parser")
comment_items = soup.select("div.name.main-name")
print(len(comment_items))

Related

Extracting text from span tag with BeautifulSoup

I am trying to get value 26.70 from this span tag:
<span class="Trsdu(0.3s) Fw(b) Fz(36px) Mb(-4px) D(ib)" data-reactid="31">26.70</span>
I tried this:
html_text2=requests.get('https://finance.yahoo.com/quote/WRD.PA?p=WRD.PA&.tsrc=fin-srch').text
soup2 = BeautifulSoup(html_text2,'lxml')
data = soup2.select_one('span.Fz(36px)').text.strip()
print(data)
but I am getting this error:
soupsieve.util.SelectorSyntaxError: Invalid character '(' position 7
line 1:
span.Fz(36px)
^

Perhaps avoid dynamic classes an look for more stable elements, and if necessary, use the relationship between those to target that node:
import requests
from bs4 import BeautifulSoup as bs
r = requests.get('https://finance.yahoo.com/quote/WRD.PA?p=WRD.PA&.tsrc=fin-srch', headers = {'User-Agent':'Mozilla/5.0'})
soup = bs(r.content, 'lxml')
print(soup.select_one('div:has(> #quote-market-notice) > span').text)

You can do this without BeautifulSoup if you want:
from selenium import webdriver
driver = webdriver.Chrome()
driver.implicitly_wait(10)
driver.get('https://finance.yahoo.com/quote/WRD.PA?p=WRD.PA&.tsrc=fin-srch')
element = driver.find_element_by_class_name("Fz\(36px\)")
print(element.text)

Get html text with Beautiful Soup

I'm trying to get the number from inside a div:
<div class="tv-symbol-price-quote__value js-symbol-last">122.7<span class="">8</span></div>
I need the 122.7 number, but I cant get it. I have tried with:
strings = soup.find("div", class_="tv-symbol-price-quote__value js-symbol-last").string
But, there are more than one element and I receive "none".
Is there a way to print the childs and get the string from childs?

Use .getText().
For example:
from bs4 import BeautifulSoup
sample_html = """
<div class="tv-symbol-price-quote__value js-symbol-last">122.7<span class="">8</span></div>
"""
soup = BeautifulSoup(sample_html, "html.parser")
strings = soup.find("div", class_="tv-symbol-price-quote__value js-symbol-last").getText()
print(strings)
Output:
122.78
Or use __next__() to get only the 122.7.
soup = BeautifulSoup(sample_html, "html.parser")
strings = soup.find("div", class_="tv-symbol-price-quote__value js-symbol-last").strings.__next__()
print(strings)
Output:
122.7

To only get the first text, search for the tag, and call the next_element method.
from bs4 import BeautifulSoup
html = """
<div class="tv-symbol-price-quote__value js-symbol-last">122.7<span class="">8</span></div>
"""
soup = BeautifulSoup(html, "html.parser")
print(
soup.find("div", class_="tv-symbol-price-quote__value js-symbol-last").next_element
)
Output:
122.7

You could use selenium to find the element and then use BS4 to parse it.
An example would be
import selenium.webdriver as WD
from selenium.webdrive.chrome.options import Options
import bs4 as B
driver = WD.Chrome()
objXpath = driver.find_element_by_xpath("""yourelementxpath""")
objHtml = objXpath.get_attribute("outerHTML")
soup = B.BeutifulSoup(objHtml, 'html.parser')
text = soup.get_text()
This code should work.
DISCLAIMER
I haven't done work w/ selenium and bs4 in a while so you might have to tweak it a little bit.

beautifulsoup can't get 'tr' in table

I'm trying to get a list of company names (e.g. 01Venture) and types (e.g. GENERAL PATERNER) from this website https://www.bvca.co.uk/Member-Directory. I'm using the code below:
import requests
from bs4 import BeautifulSoup
URL = 'https://www.bvca.co.uk/Member-Directory'
page = requests.get(URL)
soup = BeautifulSoup(page.content, 'html.parser')
print(soup.prettify())
table = soup.find('table', attrs={'id':'searchresults'})
table_body = table.find('tbody')
rows = table_body.find_all('tr')
print(rows)
And I got an empty list.

Use the selenium package, you will need to install chromedriver.
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
URL = 'https://www.bvca.co.uk/Member-Directory'
BrowserOptions = Options()
BrowserOptions.add_argument("--headless")
Browser = webdriver.Chrome(executable_path=r'chromedriver.exe', options=BrowserOptions)
Browser.get(URL)
while True:
if Browser.find_elements_by_class_name('companyName'):
break
html_source_code = Browser.execute_script("return document.body.innerHTML;")
soup = BeautifulSoup(html_source_code, 'html.parser')
x = [r.text for r in soup.find_all('h5',class_='companyName')]
print(x)
>>> ['01 Ventures', '01 Ventures', '17Capital LLP', '17Capital LLP', '1818 Venture Capital', ..., 'Zouk Capital LLP', 'Zouk Capital LLP']
The while loop waits until the company names are loaded before the html code is saved
The output was too large to put into the answer, so I could only show some of it.

How to return a list of all URL's in a YouTube playlist using Beautiful Soup?

I've tried many methods from similar questions and can't get anything to return the video URL's.
Using Chrome DevTools, the a links with hrefs for video urls are clearly in there, but when I try to return anything with an href in the format of a video url, nothing appears.
from bs4 import BeautifulSoup as bs
import requests
def getlinks(url):
r = requests.get(url)
soup = bs(r.content, features='lxml')
for link in soup.find_all("a"):
href = link.get('href')
if href.startswith('/watch?'):
print(href)
getlinks('https://www.youtube.com/playlist?list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev')

YouTube is dynamically loaded using javascript. The best way to scrape dynamic websites is to use selenium. Here is the full code to do it:
from bs4 import BeautifulSoup as bs
from selenium import webdriver
import time
def getlinks(url):
driver = webdriver.Chrome()
driver.get(url)
time.sleep(4)
soup = bs(driver.page_source, features='lxml')
for link in soup.find_all("a"):
try:
href = link.get('href')
if href.startswith('/watch?'):
print(href)
except:
pass
driver.close()
getlinks('https://www.youtube.com/playlist?list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev')
Output:
/watch?v=QktCIWrt04Q&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev
/watch?v=QktCIWrt04Q&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev
/watch?v=1PT7MP5VxDA&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev
/watch?v=QktCIWrt04Q&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=1
/watch?v=QktCIWrt04Q&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=1
/watch?v=d55aj7CPRUU&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=2
/watch?v=d55aj7CPRUU&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=2
/watch?v=V0_WnzO1hdE&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=3
/watch?v=V0_WnzO1hdE&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=3
/watch?v=y8byHSLhbcA&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=4
/watch?v=y8byHSLhbcA&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=4
/watch?v=0BKEYKDG5x0&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=5
/watch?v=0BKEYKDG5x0&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=5
/watch?v=fFB4xcH7Xik&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=6
/watch?v=fFB4xcH7Xik&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=6
/watch?v=0Y3Ln0qB7NE&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=7
/watch?v=0Y3Ln0qB7NE&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=7
/watch?v=63SP6JI4bNU&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=8
/watch?v=63SP6JI4bNU&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=8
/watch?v=eApXnJUczCI&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=9
/watch?v=eApXnJUczCI&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=9
/watch?v=0nWePxShMqk&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=10
/watch?v=0nWePxShMqk&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=10
/watch?v=z9EcZkTTMVY&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=11
/watch?v=z9EcZkTTMVY&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=11
/watch?v=ZpJIDYphWTI&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=12
/watch?v=ZpJIDYphWTI&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=12
/watch?v=FI2XIYadz1c&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=13
/watch?v=FI2XIYadz1c&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=13
/watch?v=fc4kvEElN7c&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=14
/watch?v=fc4kvEElN7c&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=14
/watch?v=6MN6Z0XBqLA&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=15
/watch?v=6MN6Z0XBqLA&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=15
/watch?v=RgWxEtOE3sU&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=16
/watch?v=RgWxEtOE3sU&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=16
/watch?v=t9U7t1NoXiw&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=17
/watch?v=t9U7t1NoXiw&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=17
/watch?v=m420JGn9yyk&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=18
/watch?v=m420JGn9yyk&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=18
/watch?v=p-Cr8Y8ZTz0&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=19
/watch?v=p-Cr8Y8ZTz0&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=19
/watch?v=g2t2qjOh-AE&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=20
/watch?v=g2t2qjOh-AE&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=20
/watch?v=qSkWUONMJSY&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=21
/watch?v=qSkWUONMJSY&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=21
/watch?v=FYMy8HaWrX4&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=22
/watch?v=FYMy8HaWrX4&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=22
/watch?v=nk5V0VbLkrI&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=23
/watch?v=nk5V0VbLkrI&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=23
/watch?v=HAsjI7wyDBg&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=24
/watch?v=HAsjI7wyDBg&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=24
/watch?v=eUKNj9VrJIU&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=25
/watch?v=eUKNj9VrJIU&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=25
/watch?v=e8Fjy01Acl0&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=26
/watch?v=e8Fjy01Acl0&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=26
/watch?v=ntz-mYI-ZBU&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=27
/watch?v=ntz-mYI-ZBU&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=27
/watch?v=kCnuXGzRO2Q&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=28
/watch?v=kCnuXGzRO2Q&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=28
/watch?v=YZWVAG7PMbo&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=29
/watch?v=YZWVAG7PMbo&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=29
/watch?v=xFIjEGwVTrI&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=30
/watch?v=xFIjEGwVTrI&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=30
/watch?v=BP9W16ay_vU&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=31
/watch?v=BP9W16ay_vU&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=31
/watch?v=tiTuSBs5k4k&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=32
/watch?v=tiTuSBs5k4k&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=32
/watch?v=_cU0eSNx1xk&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=33
/watch?v=_cU0eSNx1xk&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=33
/watch?v=xv6s5LYh5kA&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=34
/watch?v=xv6s5LYh5kA&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=34
/watch?v=0SQWvDK9_c8&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=35
/watch?v=0SQWvDK9_c8&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=35
/watch?v=1PT7MP5VxDA&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=36
/watch?v=1PT7MP5VxDA&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=36
/watch?v=1U_m3UNm93I&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=37
/watch?v=1U_m3UNm93I&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=37
/watch?v=eZlXaNaLfa8&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=38
/watch?v=eZlXaNaLfa8&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=38
/watch?v=cohVBrS9sss&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=39
/watch?v=cohVBrS9sss&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=39
/watch?v=Ulb3esvyy0I&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=40
/watch?v=Ulb3esvyy0I&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=40
/watch?v=BRiTeq0sWVo&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=41
/watch?v=BRiTeq0sWVo&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=41
/watch?v=clexityYu5k&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=42
/watch?v=clexityYu5k&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=42
/watch?v=DXGbsaU5FU0&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=43
/watch?v=DXGbsaU5FU0&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=43
/watch?v=JMl6K4-1BI0&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=44
/watch?v=JMl6K4-1BI0&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=44
/watch?v=Bry5nbhl-60&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=45
/watch?v=Bry5nbhl-60&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=45
/watch?v=KPHivuf5eqo&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=46
/watch?v=KPHivuf5eqo&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=46
/watch?v=H08DcY1tsUI&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=47
/watch?v=H08DcY1tsUI&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=47
/watch?v=19d7Ir3LGik&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=48
/watch?v=19d7Ir3LGik&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=48
/watch?v=DgDCjLd3tP4&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=49
/watch?v=DgDCjLd3tP4&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=49
/watch?v=JmjDu0Qx2GA&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=50
/watch?v=JmjDu0Qx2GA&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=50
/watch?v=Dw8kqBbRJPI&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=51
/watch?v=Dw8kqBbRJPI&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=51
/watch?v=i7DdXfbZhmQ&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=52
/watch?v=i7DdXfbZhmQ&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=52
/watch?v=O5KUFnthO4w&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=53
/watch?v=O5KUFnthO4w&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=53
/watch?v=HdrQ8qxJ8m0&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=54
/watch?v=HdrQ8qxJ8m0&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=54
/watch?v=8U9pwWqAKCE&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=55
/watch?v=8U9pwWqAKCE&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=55
/watch?v=orakWbfY86k&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=56
/watch?v=orakWbfY86k&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=56
/watch?v=J-YN9L3gvmQ&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=57
/watch?v=J-YN9L3gvmQ&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=57
/watch?v=GzzN_z_exSw&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=58
/watch?v=GzzN_z_exSw&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=58
/watch?v=CnJWxnXY7Tg&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=59
/watch?v=CnJWxnXY7Tg&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=59
/watch?v=UXteHWBdQfo&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=60
/watch?v=UXteHWBdQfo&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=60
/watch?v=OSrI0btxn_g&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=61
/watch?v=OSrI0btxn_g&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=61
/watch?v=w1YcbkJZ2aQ&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=62
/watch?v=w1YcbkJZ2aQ&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=62
/watch?v=lkisdRsjIF0&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=63
/watch?v=lkisdRsjIF0&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=63
/watch?v=zCjW5lvlLBc&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=64
/watch?v=zCjW5lvlLBc&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=64
/watch?v=pwLoCEzytyI&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=65
/watch?v=pwLoCEzytyI&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=65
/watch?v=K1mxzhZZZzA&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=66
/watch?v=K1mxzhZZZzA&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=66
/watch?v=zfAkzOEvYys&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=67
/watch?v=zfAkzOEvYys&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=67
/watch?v=yjoMEvS7vgg&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=68
/watch?v=yjoMEvS7vgg&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=68
/watch?v=fXB7p01YKxg&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=69
/watch?v=fXB7p01YKxg&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=69
/watch?v=vDr1z1Idbi8&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=70
/watch?v=vDr1z1Idbi8&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=70
/watch?v=VPwmYGteKBA&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=71
/watch?v=VPwmYGteKBA&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=71
/watch?v=nRlMGZUd5vc&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=72
/watch?v=nRlMGZUd5vc&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=72
/watch?v=97X2vjiLh8s&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=73
/watch?v=97X2vjiLh8s&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=73
/watch?v=qBW68T_PYDg&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=74
/watch?v=qBW68T_PYDg&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=74
/watch?v=xvrmcPnNQAg&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=75
/watch?v=xvrmcPnNQAg&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=75
/watch?v=RCm5pxLxTnE&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=76
/watch?v=RCm5pxLxTnE&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=76
/watch?v=hHBnPey8CCQ&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=77
/watch?v=hHBnPey8CCQ&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=77
/watch?v=ZsTy9pSLXNw&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=78
/watch?v=ZsTy9pSLXNw&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=78
/watch?v=jzKLVnDTSrs&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=79
/watch?v=jzKLVnDTSrs&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=79
/watch?v=NLoYKYudBk4&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=80
/watch?v=NLoYKYudBk4&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=80
/watch?v=HzLDetYIxTg&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=81
/watch?v=HzLDetYIxTg&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=81
/watch?v=86qvqJsd97Y&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=82
/watch?v=86qvqJsd97Y&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=82
/watch?v=cwen9S-Ckuo&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=83
/watch?v=cwen9S-Ckuo&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=83
/watch?v=2kdvdoWaoSU&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=84
/watch?v=2kdvdoWaoSU&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=84
/watch?v=VCUF0Xtop8o&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=85
/watch?v=VCUF0Xtop8o&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=85
/watch?v=tKGL4N87PMU&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=86
/watch?v=tKGL4N87PMU&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=86
/watch?v=GR_jRvqQ9Lo&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=87
/watch?v=GR_jRvqQ9Lo&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=87
/watch?v=0wEOCpLCabo&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=88
/watch?v=0wEOCpLCabo&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=88
/watch?v=Fvs5ytXZ7Cw&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=89
/watch?v=Fvs5ytXZ7Cw&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=89
/watch?v=weuz6Imoll0&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=90
/watch?v=weuz6Imoll0&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=90
/watch?v=8NEJwtWjppY&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=91
/watch?v=8NEJwtWjppY&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=91
/watch?v=7J0o12rE9FA&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=92
/watch?v=7J0o12rE9FA&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=92
/watch?v=QujbxxETYaI&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=93
/watch?v=QujbxxETYaI&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=93
/watch?v=RSRh3NtDjbc&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=94
/watch?v=RSRh3NtDjbc&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=94
/watch?v=ERQNhqghg5E&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=95
/watch?v=ERQNhqghg5E&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=95
/watch?v=Edguhs6NtM0&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=96
/watch?v=Edguhs6NtM0&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=96
/watch?v=KIPZ9EJ2LaM&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=97
/watch?v=KIPZ9EJ2LaM&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=97
/watch?v=8eLYQeB5UEk&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=98
/watch?v=8eLYQeB5UEk&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=98
/watch?v=HOzTLDzlHjk&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=99
/watch?v=HOzTLDzlHjk&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=99
/watch?v=zMqS1hzCUuA&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=100
/watch?v=zMqS1hzCUuA&list=PLU6BYY1Lu_feVbuZEscpd6xT32zCrVrev&index=100

Scraping links from buttons on a page

I am trying to scrape the links from the "box score" button on this page. The button is supposed to look like this
http://www.espn.com/nfl/boxscore?gameId=400874795
I tried to use this code to see if I could access the buttons but I cannot.
from bs4 import BeautifulSoup
import requests
url = 'http://www.espn.com/nfl/scoreboard/_/year/2016/seasontype/1/week/2'
advanced = url
r = requests.get(advanced)
data = r.text
soup = BeautifulSoup(data,"html.parser")
for link in soup.find_all('a'):
print link

As wpercy mentions in his comment, you can't do this using requests, as a suggestion you should use selenium together with Chromedriver/PhantomJS for handling the JavaScript:
from selenium import webdriver
from bs4 import BeautifulSoup
url = "http://www.espn.com/nfl/scoreboard/_/year/2016/seasontype/1/week/2"
browser = webdriver.Chrome()
browser.get(url)
html = browser.page_source
soup = BeautifulSoup(html,'html.parser')
boxList = soup.findAll('a',{'name':'&lpos=nfl:scoreboard:boxscore'})
All score buttons's a tag have the attribute name = &lpos=nfl:scoreboard:boxscore, so we first use .findAll and now a simple list comprehension can extract each href attribute:
>>> links = [box['href'] for box in boxList]
>>> links
['/nfl/boxscore?gameId=400874795', '/nfl/boxscore?gameId=400874854', '/nfl/boxscore?gameId=400874753', '/nfl/boxscore?gameId=400874757', '/nfl/boxscore?gameId=400874772', '/nfl/boxscore?gameId=400874777', '/nfl/boxscore?gameId=400874767', '/nfl/boxscore?gameId=400874812', '/nfl/boxscore?gameId=400874761', '/nfl/boxscore?gameId=400874764', '/nfl/boxscore?gameId=400874781', '/nfl/boxscore?gameId=400874796', '/nfl/boxscore?gameId=400874750', '/nfl/boxscore?gameId=400873867', '/nfl/boxscore?gameId=400874775', '/nfl/boxscore?gameId=400874798']

here is the solution i did , and it scrapes all the link which are there on the url you have provided in your answer . you can check it out
# from BeautifulSoup import *
from bs4 import BeautifulSoup
# import requests
import urllib
url = 'http://www.espn.com/nfl/scoreboard/_/year/2016/seasontype/1/week/2'
# advanced = url
html = urllib.urlopen(url).read()
# r = requests.get(html)
# data = r.text
soup = BeautifulSoup(html)
tags = soup('a')
# for link in soup.find_all('a'):
for i,tag in enumerate(tags):
# print tag;
print i;
ans = tag.get('href',None)
print ans;
print "\n";

The answer from Gopal Chitalia didn't work for me, so I decided to post the working one (for python 3.6.5)
# from BeautifulSoup import *
from bs4 import BeautifulSoup
# import requests
import urllib
url = 'http://www.espn.com/nfl/scoreboard/_/year/2016/seasontype/1/week/2'
# advanced = url
html = urllib.request.urlopen(url)
# urlopen(url).read()
# r = requests.get(html)
# data = r.text
soup = BeautifulSoup(html)
tags = soup('a')
# for link in soup.find_all('a'):
for i,tag in enumerate(tags):
# print tag;
print (i);
ans = tag.get('href',None)
print (ans);
print ("\n");

We Keep Coding

Python is a programming language that lets you work quickly and integrate systems more effectively.

BeautifulSoup can't find element class in HTML - python

Related

Extracting text from span tag with BeautifulSoup

Get html text with Beautiful Soup

beautifulsoup can't get 'tr' in table

How to return a list of all URL's in a YouTube playlist using Beautiful Soup?

Scraping links from buttons on a page

Categories

Resources