import requests
from bs4 import BeautifulSoup
results = requests.get("https://en.wikipedia.org/wiki/List_of_multiple_Olympic_gold_medalists")
src = results.content
soup = BeautifulSoup(src, 'lxml')
trs = soup.find_all("tr")
for tr in trs:
print(tr.text)
This is the code I write for the scraping table from the page "https://en.wikipedia.org/wiki/List_of_multiple_Olympic_gold_medalists"
If I am only targeting the table in the session "List of most Olympic gold medals over career", how can I specify the table I need? There are 2 sortable jquery-tablesorter so I cannot use the class attribute to select the table I needed.
One more question, if I know that the page I am scraping contains a lot of tables and the one I need always have 10 td in 1 row, can I have something like
If len(td) == 10:
print(tr)
to extract the data I wanted
Update on code:
from bs4 import BeautifulSoup
results = requests.get("https://en.wikipedia.org/wiki/List_of_multiple_Olympic_gold_medalists")
src = results.content
soup = BeautifulSoup(src, 'lxml')
tbs = soup.find("tbody")
trs = tbs.find_all("tr")
for tr in trs:
print(tr.text)
I have one of the solution, not a good one, just to extract the first table from the page which is the one I needed, any suggestion/ improvement are welcomed!
Thank you.
To only get the first table you can use a CSS Selector nth-of-type(1):
import requests
from bs4 import BeautifulSoup
URL = "https://en.wikipedia.org/wiki/List_of_multiple_Olympic_gold_medalists"
soup = BeautifulSoup(requests.get(URL).content, "html.parser")
table = soup.select_one("table.wikitable:nth-of-type(1)")
trs = table.find_all("tr")
for tr in trs:
print(tr.text)
Related
I want to check span class (glyphicon icon icon-positive = True) and if so take the value from tr rel (/reestr/clients/233/members/3567150). How i can do this?
I don't understand how to access this data.
from bs4 import BeautifulSoup
import requests
url = 'http://reestr.nostroy.ru/reestr?m.fulldescription=&m.shortdescription=&m.inn=6674374250&m.ogrnip=&bms.id=&bmt.id=&u.registrationnumber='
html = requests.get(url)
soup = BeautifulSoup(html.content, 'html.parser')
news = []
new_news = []
news = soup.findAll('table', class_='items table table-selectable-row table-striped')
Give this a go:
news = soup.find('table', class_='items table table-selectable-row table-striped')
for tr in news.find_all('tr'):
if tr.find('span',class_='glyphicon icon icon-positive'):
print(tr['rel'])
Note I changed the way you find news (with .find instead of .find_all) as there is only one object matching that condition.
I am new to web scraping and I'm trying to scrape the "statistics" page of yahoo finance for AAPL. Here's the link: https://finance.yahoo.com/quote/AAPL/key-statistics?p=AAPL
Here is the code I have so far...
from bs4 import BeautifulSoup
from requests import get
url = 'https://finance.yahoo.com/quote/AAPL/key-statistics?p=AAPL'
response = get(url)
soup = BeautifulSoup(response.text, 'html.parser')
stock_data = soup.find_all("table")
for stock in stock_data:
print(stock.text)
When I run that, I return all of the table data on the page. However, I only want specific data from each table (e.g. "Market Cap", "Revenue", "Beta").
I tried messing around with the code by doing print(stock[1].text) to see if I could limit the amount of data returned to just the second value in each table but that returned an error message. Am I on the right track by using BeautifulSoup or do I need to use a completely different library? What would I have to do in order to only return particular data and not all of the table data on the page?
Examining the HTML-code gives you the best idea of how BeautifulSoup will handle what it sees.
The web page seems to contain several tables, which in turn contain the information you are after. The tables follow a certain logic.
First scrape all the tables on the web page, then find all the table rows (<tr>) and the table data (<td>) that those rows contain.
Below is one way of achieving this. I even threw in a function to print only a specific measurement.
from bs4 import BeautifulSoup
from requests import get
url = 'https://finance.yahoo.com/quote/AAPL/key-statistics?p=AAPL'
response = get(url)
soup = BeautifulSoup(response.text, 'html.parser')
stock_data = soup.find_all("table")
# stock_data will contain multiple tables, next we examine each table one by one
for table in stock_data:
# Scrape all table rows into variable trs
trs = table.find_all('tr')
for tr in trs:
# Scrape all table data tags into variable tds
tds = tr.find_all('td')
# Index 0 of tds will contain the measurement
print("Measure: {}".format(tds[0].get_text()))
# Index 1 of tds will contain the value
print("Value: {}".format(tds[1].get_text()))
print("")
def get_measurement(table_array, measurement):
for table in table_array:
trs = table.find_all('tr')
for tr in trs:
tds = tr.find_all('td')
if measurement.lower() in tds[0].get_text().lower():
return(tds[1].get_text())
# print only one measurement, e.g. operating cash flow
print(get_measurement(stock_data, "operating cash flow"))
Although this isn't Yahoo Finance, you can do something very similar like this...
import requests
from bs4 import BeautifulSoup
base_url = 'https://finviz.com/screener.ashx?v=152&o=price&t=MSFT,AAPL,SBUX,S,GOOG&o=price&c=0,1,2,3,4,5,6,7,8,9,25,63,64,65,66,67'
html = requests.get(base_url)
soup = BeautifulSoup(html.content, "html.parser")
main_div = soup.find('div', attrs = {'id':'screener-content'})
light_rows = main_div.find_all('tr', class_="table-light-row-cp")
dark_rows = main_div.find_all('tr', class_="table-dark-row-cp")
data = []
for rows_set in (light_rows, dark_rows):
for row in rows_set:
row_data = []
for cell in row.find_all('td'):
val = cell.a.get_text()
row_data.append(val)
data.append(row_data)
# sort rows to maintain original order
data.sort(key=lambda x: int(x[0]))
import pandas
pandas.DataFrame(data).to_csv("C:\\your_path\\AAA.csv", header=False)
This is a nice substitute in case Yahoo decided to depreciate more of the functionality of their API. I know they cut out a lot of things (mostly historical quotes) a couple years ago. It was sad to see that go away.
I want to extract text from th tags in a table so I can print a list of metro stations from a table in a Wikipedia page. I only need text from a certain table (there are two of them in the page)
import urllib.request
url = "https://en.wikipedia.org/wiki/List_of_London_Underground_stations"
page = urllib.request.urlopen(url)
from bs4 import BeautifulSoup
soup = BeautifulSoup(page, "lxml")
stations_table = soup.find("table", class_= "wikitable sortable plainrowheaders")
stations_table
for i in soup.find_all('th', stations_table):
print(i.text)
I can get the table stored in the stations_table variable but cannot print the text in th tags within the wikitable sortable plainrowheaders table. While it does print the station name, it also prints the headers:
Station
Local authority
Zone(s)[†]
Opened[4]
Main lineopened
Usage[5]
How can I filter those out?
It shows all th in table - not only stations but also headers like Stations, Lines
To skip it I search all tr, skip first row and then I search th in every row
for i in stations_table.find_all('tr')[1:]
print(i.find('th').text.strip())
Full code
import urllib.request
from bs4 import BeautifulSoup
url = "https://en.wikipedia.org/wiki/List_of_London_Underground_stations"
page = urllib.request.urlopen(url)
soup = BeautifulSoup(page, "html.parser")
stations_table = soup.find("table", class_= "wikitable sortable plainrowheaders")
for i in stations_table.find_all('tr')[1:]:
print(i.find('th').text.strip())
#print(i.th.text.strip())
for i in soup.find_all('th', stations_table):
searches for all the table headings and the table rows. What can be done for this, is to extract all the rows and start printing from the second row (ignoring the title's row) as below
for i in stations_table.find_all('tr')[1:]:
print(i.find('th').text)
I am trying to scrape the bitcoin price off of coinbase and cannot find the proper syntax. When I run the program (without the line with question marks) I get the block of html that I need, but I don't know how to narrow down and retrieve the price itself. Any help appreciated, thanks.
import requests
from bs4 import BeautifulSoup
url = 'https://www.coinbase.com/charts'
data = requests.get(url)
nicedata = data.text
soup = BeautifulSoup(nicedata, 'html.parser')
prettysoup = soup.prettify()
bitcoin = soup.find('h4', {'class':
'Header__StyledHeader-sc-1q6y56a-0 hZxUBM
TextElement__Spacer-sc-18l8wi5-0 hpeTzd'})
price = bitcoin.find('???')
print(price)
The attached image contains the html
To get text from item:
price = bitcoin.text
But this page has many items <h4> with this class but find() gets only first one and it has text Bitcoin, not price from your image. You may need find_all() to get list with all items and then you can use index [index] or slicing [start:end] to get some items, or you can use for-loop to work with every item on list.
import requests
from bs4 import BeautifulSoup
url = 'https://www.coinbase.com/charts'
r = requests.get(url)
soup = BeautifulSoup(r.text, 'html.parser')
all_h4 = soup.find_all('h4', {'class': 'Header__StyledHeader-sc-1q6y56a-0 hZxUBM TextElement__Spacer-sc-18l8wi5-0 hpeTzd'})
for h4 in all_h4:
print(h4.text)
It can be easier to work with data if you keep it in list of list or array or DataFrame. But to create list of lists it would be easier to find rows <tr> and inside every row search <h4>
import requests
from bs4 import BeautifulSoup
url = 'https://www.coinbase.com/charts'
r = requests.get(url, headers=headers)
soup = BeautifulSoup(r.text, 'html.parser')
all_tr = soup.find_all('tr')
data = []
for tr in all_tr:
row = []
for h4 in tr.find_all('h4'):
row.append(h4.text)
if row: # skip empty row
data.append(row)
for row in data:
print(row)
It doesn't need class to get all h4.
BTW: This page uses JavaScript to append new rows when you scroll page but requests and BeautifulSoup can't run JavaScript - so if you will need all rows then you may need Selenium to control web browser which runs JavaScript
I am trying to isolate the Location column and then eventually get it to output to a database file. My code is as follows:
import urllib
import urllib2
from bs4 import BeautifulSoup
url = "http://en.wikipedia.org/wiki/List_of_ongoing_armed_conflicts"
response = urllib2.urlopen(url)
html = response.read()
soup = BeautifulSoup(html)
trs = soup.find_all('td')
for tr in trs:
for link in tr.find_all('a'):
fulllink = link.get ('href')
tds = tr.find_all("tr")
location = str(tds[3].get_text())
print location
but I always get 1 of 2 errors either list being out of range or exit code '0'. I am uncertain on beautfulsoup as I am trying to learn it so any help is appreciated thanks !
There is an easier way to locate the Location column. Use a table.wikitable tr CSS Selector, find all td elements for every row and get the 4th td by index.
Besides, if there are multiple locations inside a cell, you need to treat them separately:
import urllib2
from bs4 import BeautifulSoup
url = "http://en.wikipedia.org/wiki/List_of_ongoing_armed_conflicts"
soup = BeautifulSoup(urllib2.urlopen(url))
for row in soup.select('table.wikitable tr'):
cells = row.find_all('td')
if cells:
for text in cells[3].find_all(text=True):
text = text.strip()
if text:
print text
Prints:
Afghanistan
Nigeria
Cameroon
Niger
Chad
...
Iran
Nigeria
Mozambique
You simply swap td and tr balises in your code. And be careful with str() function because you can have unicode string in your web page that cannot be converted in simple ascii string. Your code should be:
import urllib
import urllib2
from bs4 import BeautifulSoup
url = "http://en.wikipedia.org/wiki/List_of_ongoing_armed_conflicts"
response = urllib2.urlopen(url)
html = response.read()
soup = BeautifulSoup(html)
trs = soup.find_all('tr') # 'tr' instead of td
for tr in trs:
for link in tr.find_all('a'):
fulllink = link.get ('href')
tds = tr.find_all("td") # 'td' instead of td
location = tds[3].get_text() # remove of str function
print location
And voilà!!