I have a column with 1,5k lines and each line has this structure:
" [{'id': 4099, 'name': 'xxxxxxxx + 30 filter', 'product_id': 6546, 'variation_id': 3352, 'quantity': 1, 'tax_class': '', 'subtotal': '110.89', 'subtotal_tax': '0.00', 'total': '29.90', 'total_tax': '0.00', 'taxes': [], 'meta_data': [{'id': 39083, 'key': 'pa_size', 'value': 'l', 'display_key': 'Size', 'display_value': 'L'}, {'id': 39094, 'key': '_reduced_stock', 'value': '1', 'display_key': '_reduced_stock', 'display_value': '1'}], 'sku': 'FS00055.L', 'price': 29.9, 'parent_name': 'xxxxxxxx + 30 filter'}] "
I want to get to a result in which the keys of the inner dictionary become new columns and the values are copied underneath. For example:
id name ......
4099 xxxxxxxx + 30 filter ......
I've tried:
import ast
# Acess only the first line and try to split each into columns
li_column = my_df.loc[0,'line_items']
li_column = ast.literal_eval(li_column)
Then I was able to get into a list with 1 item that has a dictionary inside, now I'm stuck.
I can't convert your string with ast but you could use .apply() to do it for all rows
df['line_items'] = df['line_items'].apply(dirtyjson.loads)
Later you could get dictionary from list
df['line_items'] = df['line_items'].str[0]
And later you can use again .apply() with pd.Series to create new DataFrame from dictionaries.
new_df = df['line_items'].apply(pd.Series)
As I said ast doesn't work for me with your strings. Standard json has also problem to convert it. But module dirtyjson convert it correctly.
import pandas as pd
df = pd.DataFrame({
'line_items': [
" [{'id': 4099, 'name': 'xxxxxxxx + 30 filter', 'product_id': 6546, 'variation_id': 3352, 'quantity': 1, 'tax_class': '', 'subtotal': '110.89', 'subtotal_tax': '0.00', 'total': '29.90', 'total_tax': '0.00', 'taxes': [], 'meta_data': [{'id': 39083, 'key': 'pa_size', 'value': 'l', 'display_key': 'Size', 'display_value': 'L'}, {'id': 39094, 'key': '_reduced_stock', 'value': '1', 'display_key': '_reduced_stock', 'display_value': '1'}], 'sku': 'FS00055.L', 'price': 29.9, 'parent_name': 'xxxxxxxx + 30 filter'}] ",
" [{'id': 4199, 'name': 'xxxxxxxx + 30 filter', 'product_id': 6546, 'variation_id': 3352, 'quantity': 1, 'tax_class': '', 'subtotal': '110.89', 'subtotal_tax': '0.00', 'total': '29.90', 'total_tax': '0.00', 'taxes': [], 'meta_data': [{'id': 39083, 'key': 'pa_size', 'value': 'l', 'display_key': 'Size', 'display_value': 'L'}, {'id': 39094, 'key': '_reduced_stock', 'value': '1', 'display_key': '_reduced_stock', 'display_value': '1'}], 'sku': 'FS00055.L', 'price': 29.9, 'parent_name': 'xxxxxxxx + 30 filter'}] ",
]
})
#import ast
#df['line_items'] = df['line_items'].apply(ast.literal_eval)
import dirtyjson
df['line_items'] = df['line_items'].apply(dirtyjson.loads)
df['line_items'] = df['line_items'].str[0]
new_df = df['line_items'].apply(pd.Series)
print(new_df)
Result:
id name ... price parent_name
0 4099 xxxxxxxx + 30 filter ... 29.9 xxxxxxxx + 30 filter
1 4199 xxxxxxxx + 30 filter ... 29.9 xxxxxxxx + 30 filter
[2 rows x 15 columns]
EDIT:
If you need to add it to existing dataframe
df = df.join(new_df)
# remove old column
del df['line_items']
print(df)
EDIT:
If every list may have more dictionaries then you can use explode() instead of str[0] - and it will put every dictionary in separated row.
df = df.explode('line_items').reset_index(drop=True) # extract from list
import pandas as pd
df = pd.DataFrame({
#'A': ['123', '456', '789'],
'line_items': [
""" [{'id': 4099, 'name': 'xxxxxxxx + 30 filter', 'product_id': 6546, 'variation_id': 3352, 'quantity': 1, 'tax_class': '', 'subtotal': '110.89', 'subtotal_tax': '0.00', 'total': '29.90', 'total_tax': '0.00', 'taxes': [], 'meta_data': [{'id': 39083, 'key': 'pa_size', 'value': 'l', 'display_key': 'Size', 'display_value': 'L'}, {'id': 39094, 'key': '_reduced_stock', 'value': '1', 'display_key': '_reduced_stock', 'display_value': '1'}], 'sku': 'FS00055.L', 'price': 29.9, 'parent_name': 'xxxxxxxx + 30 filter'},
{'id': 5099, 'name': 'xxxxxxxx + 30 filter', 'product_id': 6546, 'variation_id': 3352, 'quantity': 1, 'tax_class': '', 'subtotal': '110.89', 'subtotal_tax': '0.00', 'total': '29.90', 'total_tax': '0.00', 'taxes': [], 'meta_data': [{'id': 39083, 'key': 'pa_size', 'value': 'l', 'display_key': 'Size', 'display_value': 'L'}, {'id': 39094, 'key': '_reduced_stock', 'value': '1', 'display_key': '_reduced_stock', 'display_value': '1'}], 'sku': 'FS00055.L', 'price': 29.9, 'parent_name': 'xxxxxxxx + 30 filter'}] """,
""" [{'id': 4100, 'name': 'xxxxxxxx + 30 filter', 'product_id': 6546, 'variation_id': 3352, 'quantity': 1, 'tax_class': '', 'subtotal': '110.89', 'subtotal_tax': '0.00', 'total': '29.90', 'total_tax': '0.00', 'taxes': [], 'meta_data': [{'id': 39083, 'key': 'pa_size', 'value': 'l', 'display_key': 'Size', 'display_value': 'L'}, {'id': 39094, 'key': '_reduced_stock', 'value': '1', 'display_key': '_reduced_stock', 'display_value': '1'}], 'sku': 'FS00055.L', 'price': 29.9, 'parent_name': 'xxxxxxxx + 30 filter'},
{'id': 5100, 'name': 'xxxxxxxx + 30 filter', 'product_id': 6546, 'variation_id': 3352, 'quantity': 1, 'tax_class': '', 'subtotal': '110.89', 'subtotal_tax': '0.00', 'total': '29.90', 'total_tax': '0.00', 'taxes': [], 'meta_data': [{'id': 39083, 'key': 'pa_size', 'value': 'l', 'display_key': 'Size', 'display_value': 'L'}, {'id': 39094, 'key': '_reduced_stock', 'value': '1', 'display_key': '_reduced_stock', 'display_value': '1'}], 'sku': 'FS00055.L', 'price': 29.9, 'parent_name': 'xxxxxxxx + 30 filter'}] """,
""" [{'id': 4101, 'name': 'xxxxxxxx + 30 filter', 'product_id': 6546, 'variation_id': 3352, 'quantity': 1, 'tax_class': '', 'subtotal': '110.89', 'subtotal_tax': '0.00', 'total': '29.90', 'total_tax': '0.00', 'taxes': [], 'meta_data': [{'id': 39083, 'key': 'pa_size', 'value': 'l', 'display_key': 'Size', 'display_value': 'L'}, {'id': 39094, 'key': '_reduced_stock', 'value': '1', 'display_key': '_reduced_stock', 'display_value': '1'}], 'sku': 'FS00055.L', 'price': 29.9, 'parent_name': 'xxxxxxxx + 30 filter'},
{'id': 5101, 'name': 'xxxxxxxx + 30 filter', 'product_id': 6546, 'variation_id': 3352, 'quantity': 1, 'tax_class': '', 'subtotal': '110.89', 'subtotal_tax': '0.00', 'total': '29.90', 'total_tax': '0.00', 'taxes': [], 'meta_data': [{'id': 39083, 'key': 'pa_size', 'value': 'l', 'display_key': 'Size', 'display_value': 'L'}, {'id': 39094, 'key': '_reduced_stock', 'value': '1', 'display_key': '_reduced_stock', 'display_value': '1'}], 'sku': 'FS00055.L', 'price': 29.9, 'parent_name': 'xxxxxxxx + 30 filter'}] """,
]
})
import dirtyjson
df['line_items'] = df['line_items'].apply(dirtyjson.loads)
#df['line_items'] = df['line_items'].str[0] # extract first item from list
df = df.explode('line_items').reset_index(drop=True) # extract all items from list
new_df = df['line_items'].apply(pd.Series)
print(new_df)
df = df.join(new_df)
del df['line_items']
print(df)
id name ... price parent_name
0 4099 xxxxxxxx + 30 filter ... 29.9 xxxxxxxx + 30 filter
1 5099 xxxxxxxx + 30 filter ... 29.9 xxxxxxxx + 30 filter
2 4100 xxxxxxxx + 30 filter ... 29.9 xxxxxxxx + 30 filter
3 5100 xxxxxxxx + 30 filter ... 29.9 xxxxxxxx + 30 filter
4 4101 xxxxxxxx + 30 filter ... 29.9 xxxxxxxx + 30 filter
5 5101 xxxxxxxx + 30 filter ... 29.9 xxxxxxxx + 30 filter
[6 rows x 15 columns]
I am trying to get price info using BS4 with Python.
This is the <p> class from which I am trying to extract the value of '$3245'.
<div class="s1fqyqkq-3 bDmzjk">
<div class="s1fqyqkq-4 ckAXTq">Price Info</div>
<p class="s1fqyqkq-5 dekHBg">$3245</p>
</div>
And this is the code I am trying to use to get the data, but it's returning "None"
headers = {
"Connection": "keep-alive",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36"
}
req = requests.get(url, headers=headers)
soup = bs4.BeautifulSoup(req.text, 'html.parser')
result = soup.find("div", {"class": "s1fqyqkq-5 dekHBg"})
print(result)
I tried all suggestions, but I am still I receiving "None" as a result.
I tried the suggestion here to no avail. Any idea what I am missing?
While cooking your soup taste it to ensure it comes with all expected ingridients.
Main issue, content is created dynamically by javascript so you wont find your information that way, cause requests do not act like a browser and is not able to render all information.
To get informations take a look at the script variables and extract the JSON strings.
import json, requests
url = 'https://www.cimri.com/cep-telefonlari/en-ucuz-apple-iphone-11-64gb-4gb-ram-6-1-inc-12mp-akilli-cep-telefonu-siyah-fiyatlari,337340793'
jsonData = json.loads(re.search(r"window.productDetails = (.*?);", requests.get(url).text).group(1))
jsonData
Output
{'id': '337340793', 'path': '/cep-telefonlari/en-ucuz-apple-iphone-11-64gb-4gb-ram-6-1-inc-12mp-akilli-cep-telefonu-siyah-fiyatlari,337340793', 'itemType': 0, 'mainId': 337341638, 'title': 'Apple iPhone 11 64GB 4GB Ram 6.1 inç 12MP Akıllı Cep Telefonu Siyah', 'offers': [{'id': '506482765', 'price': 13135.85, 'title': 'Yeni Apple iPhone 11 (64\xa0GB) - Siyah', 'shipping': {'fee': 0, 'day': 0}, 'merchant': {'id': '10370', 'url': 'https://www.amazon.com.tr', 'seller': 'Amazon.com.tr', 'slogan': 'Amazon.com.tr uygulamasında ilk 150 TL ve üzeri alışverişe 40 TL indirim ', 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655495611798}, {'id': '894741295', 'price': 13149, 'title': 'iPhone 11 64 GB Aksesuarsız Kutu Siyah', 'shipping': {'fee': 0, 'day': 3}, 'merchant': {'id': '10109', 'url': 'https://www.trendyol.com/', 'seller': 'Trendyol', 'slogan': "Tüm İhtiyaçların Trendyol'da, Tek Tıkla Kapında!", 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': True, 'feedLastProcess': 1655493405188}, {'id': '887498622', 'price': 13249, 'title': 'Apple iPhone 11 64 GB Cep Telefonu Siyah', 'shipping': {'fee': 0, 'day': 2}, 'merchant': {'id': '10416', 'url': 'https://www.a101.com.tr/', 'seller': None, 'slogan': 'A101|Harca Harca Bitmez', 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655489830306}, {'id': '767094934', 'price': 13299, 'title': 'iPhone 11 64 GB Siyah', 'shipping': {'fee': 0, 'day': None}, 'merchant': {'id': '4093', 'url': 'http://www.turkcell.com.tr', 'seller': None, 'slogan': None, 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655494357178}, {'id': '855428213', 'price': 13350, 'title': 'iPhone 11 64 GB Siyah (Apple Türkiye Garantili) - (Aksesuarsız Kutu)', 'shipping': {'fee': 0, 'day': 0}, 'merchant': {'id': '12192', 'url': 'https://www.needion.com/', 'seller': None, 'slogan': 'Alan Memnun, Satan Memnun', 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': True, 'feedLastProcess': 1655495321932}, {'id': '914643935', 'price': 13379, 'title': 'Apple iPhone 11 64GB Akıllı Cep Telefonu Siyah ( Apple Türkiye Garantili )', 'shipping': {'fee': 0, 'day': 1}, 'merchant': {'id': '12665', 'url': 'https://wnt.com.tr/', 'seller': None, 'slogan': None, 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655494393339}, {'id': '491596021', 'price': 13398.99, 'title': 'iPhone 11 64 GB - Siyah - Aksesuarsız Kutu', 'shipping': {'fee': None, 'day': 0}, 'merchant': {'id': '1497', 'url': 'http://www.hepsiburada.com', 'seller': 'Hepsiburada', 'slogan': 'Türkiye’de İlk Defa Alışveriş Kredisi Burada', 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655495861494}, {'id': '1027767764', 'price': 13459, 'title': 'Apple iPhone 11 64GB Siyah Cep Telefonu (Apple Türkiye Garantili) (Siyah)', 'shipping': {'fee': 0, 'day': 1}, 'merchant': {'id': '5825', 'url': 'https://www.vodafone.com.tr/', 'seller': 'GENÇPA', 'slogan': 'Vodafone Her Şey Yanımda Alışverişlerinizde Kargo Bedava, Vodafone’lulara İlk Siparişe 15GB Hediye!', 'onlyApp': True}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655495537182}, {'id': '952479181', 'price': 13499, 'title': 'iPhone 11 64 GB Aksesuarsız Kutu Siyah', 'shipping': {'fee': 0, 'day': 0}, 'merchant': {'id': '10109', 'url': 'https://www.trendyol.com/', 'seller': 'Teknosa', 'slogan': "Tüm İhtiyaçların Trendyol'da, Tek Tıkla Kapında!", 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655493405188}, {'id': '1032790438', 'price': 13499, 'title': 'Apple iPhone 11 64 GB Siyah (Apple Türkiye Garantili) (Siyah)', 'shipping': {'fee': 0, 'day': 1}, 'merchant': {'id': '5825', 'url': 'https://www.vodafone.com.tr/', 'seller': 'CepHane Teknoloji', 'slogan': 'Vodafone Her Şey Yanımda Alışverişlerinizde Kargo Bedava, Vodafone’lulara İlk Siparişe 15GB Hediye!', 'onlyApp': True}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655495537182}, {'id': '1063638766', 'price': 13509, 'title': 'Apple iPhone 11 64 GB (Apple Türkiye Garantili) siyah', 'shipping': {'fee': 0, 'day': 2}, 'merchant': {'id': '7000', 'url': 'http://www.n11.com', 'seller': 'cokteknolojik', 'slogan': 'Hayat Sana Gelir', 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655495952093}, {'id': '930481030', 'price': 13559, 'title': 'iPhone 11 64GB Black', 'shipping': {'fee': 0, 'day': 1}, 'merchant': {'id': '13105', 'url': 'https://www.inbox.com.tr/', 'seller': None, 'slogan': None, 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655491323973}, {'id': '1007434732', 'price': 13633, 'title': 'Apple iPhone 11 64 GB (Apple Türkiye Garantili) siyah', 'shipping': {'fee': 0, 'day': 1}, 'merchant': {'id': '7000', 'url': 'http://www.n11.com', 'seller': 'yukseliletisim', 'slogan': 'Hayat Sana Gelir', 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655495952093}, {'id': '912636803', 'price': 13699, 'title': 'Apple iPhone 11 64GB Siyah - MHDA3TU/A', 'shipping': {'fee': 0, 'day': 1}, 'merchant': {'id': '11403', 'url': 'https://gurgencler.com.tr/', 'seller': None, 'slogan': '10.000TL üzeri alışverişinize 500TL Bankkart Lira hediye!', 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655492821318}, {'id': '974603297', 'price': 13779, 'title': 'IPhone 11 64 GB Siyah', 'shipping': {'fee': 0, 'day': 0}, 'merchant': {'id': '12938', 'url': 'https://www.cepux.com.tr/', 'seller': None, 'slogan': 'Cepux Kazandırır!', 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655495878822}, {'id': '960442769', 'price': 13779, 'title': 'Apple iPhone 11 64 GB Aksesuarsız Kutu Siyah (Siyah)', 'shipping': {'fee': 0, 'day': 1}, 'merchant': {'id': '5825', 'url': 'https://www.vodafone.com.tr/', 'seller': 'Tokdemir Bilişim Teknolojileri', 'slogan': 'Vodafone Her Şey Yanımda Alışverişlerinizde Kargo Bedava, Vodafone’lulara İlk Siparişe 15GB Hediye!', 'onlyApp': True}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655495537182}, {'id': '965671448', 'price': 13789, 'title': 'iPhone 11 64 GB Siyah', 'shipping': {'fee': 0, 'day': None}, 'merchant': {'id': '4093', 'url': 'http://www.turkcell.com.tr', 'seller': None, 'slogan': None, 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655494357178}, {'id': '808307738', 'price': 13798, 'title': 'iPhone 11 64 GB Siyah', 'shipping': {'fee': 0, 'day': None}, 'merchant': {'id': '4093', 'url': 'http://www.turkcell.com.tr', 'seller': None, 'slogan': None, 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655494357178}, {'id': '1059226040', 'price': 13799, 'title': 'Apple iPhone 11 64 GB (Apple Türkiye Garantili) siyah', 'shipping': {'fee': 0, 'day': 1}, 'merchant': {'id': '7000', 'url': 'http://www.n11.com', 'seller': 'integrabilisim', 'slogan': 'Hayat Sana Gelir', 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655495952093}, {'id': '734101431', 'price': 13799, 'title': 'Iphone 11 64 Gb Siyah Aksesuarsız Kutu', 'shipping': {'fee': 0, 'day': 1}, 'merchant': {'id': '11512', 'url': 'https://www.pttavm.com', 'seller': 'BittiBitiyor', 'slogan': None, 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655495542557}, {'id': '765227763', 'price': 14484, 'title': 'Apple iPhone 11 64Gb Akıllı Telefon Siyah', 'shipping': {'fee': 0, 'day': 0}, 'merchant': {'id': '46', 'url': 'http://www.teknosa.com/', 'seller': 'Bittibitiyor', 'slogan': "Sepette 2.500 TL'ye Varan İndirimler", 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655489853593}, {'id': '765957525', 'price': 14499, 'title': 'iPhone 11 64 Gb Siyah', 'shipping': {'fee': 15, 'day': 1}, 'merchant': {'id': '11375', 'url': 'https://www.vatanbilgisayar.com/', 'seller': None, 'slogan': None, 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655489535355}, {'id': '765434969', 'price': 14499, 'title': 'APPLE iPhone 11 64GB Akıllı Telefon Siyah', 'shipping': {'fee': 0, 'day': None}, 'merchant': {'id': '4230', 'url': 'http://www.mediamarkt.com.tr/', 'seller': None, 'slogan': '250 TL ve üzeri alışverişlerde geçerli ücretsiz kargo', 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655489226707}, {'id': '765500161', 'price': 14549.03, 'title': 'Apple İphone 11 64Gb Siyah - Mhda3tu/A 194252097335', 'shipping': {'fee': 0, 'day': 1}, 'merchant': {'id': '10052', 'url': 'http://www.troyestore.com/', 'seller': None, 'slogan': 'Apple Yetkili Satıcı, Apple Yetkili Teknik Servis', 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655493437121}, {'id': '765498940', 'price': 14999, 'title': 'Apple iPhone 11 64 Gb Siyah Cep Telefonu', 'shipping': {'fee': 0, 'day': None}, 'merchant': {'id': '10313', 'url': 'https://www.migros.com.tr/', 'seller': None, 'slogan': '250 TL ve Üzeri Ücretsiz Teslimat', 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655488948137}, {'id': '765987003', 'price': 14999, 'title': 'Apple iPhone 11 64GB Siyah Yeni Cep Telefonu', 'shipping': {'fee': 0, 'day': 2}, 'merchant': {'id': '10604', 'url': 'https://www.beko.com.tr/', 'seller': None, 'slogan': 'Ücretsiz ve Hızlı Teslimat ile Beko Kapında!', 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655492802285}, {'id': '765350098', 'price': 14999, 'title': 'Apple iPhone 11 64GB Siyah Yeni Cep Telefonu', 'shipping': {'fee': 0, 'day': 2}, 'merchant': {'id': '10591', 'url': 'https://www.arcelik.com.tr/', 'seller': None, 'slogan': 'Ücretsiz ve Hızlı Teslimat ile Arçelik Kapında!', 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655492539972}, {'id': '765171664', 'price': 14999, 'title': 'Apple iPhone 11 64GB Siyah (New Edt)', 'shipping': {'fee': 0, 'day': None}, 'merchant': {'id': '9334', 'url': 'http://www.evkur.com.tr/', 'seller': None, 'slogan': 'Hiç Peşinatsız Kredi Kartsız 24 Aya Varan Taksitlerle', 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655491644137}, {'id': '806067367', 'price': 15292.64, 'title': 'İphone 11 64 Gb', 'shipping': {'fee': 0, 'day': 0}, 'merchant': {'id': '5164', 'url': 'http://www.gittigidiyor.com/', 'seller': 'gncpazar', 'slogan': "Yapı Kredi World'e Özel 50 TL'ye Varan Worldpuan", 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655495287185}, {'id': '1068272963', 'price': 15599, 'title': 'İphone 11 64 Gb Siyah(kvk Garantili)', 'shipping': {'fee': 0, 'day': 3}, 'merchant': {'id': '5164', 'url': 'http://www.gittigidiyor.com/', 'seller': 'AYAZGRUP', 'slogan': "Yapı Kredi World'e Özel 50 TL'ye Varan Worldpuan", 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655495287185}, {'id': '793747418', 'price': 17591.37, 'title': 'Apple IPHONE 11 64GB Apple IPHONE 11 64GB Cep Telefonu (Apple Türkiye Garantili)', 'shipping': {'fee': 25, 'day': None}, 'merchant': {'id': '11898', 'url': 'https://www.turkuaztrade.com/', 'seller': None, 'slogan': 'Simply The Best / Kısaca En İyisi', 'onlyApp': False}, 'unitPrice': None, 'numbers': [], 'sponsored': False, 'feedLastProcess': 1655494006222}], 'imageIds': [219571586, 219571585, 219571582, 219571577, 219575009], 'review': {'count': 95, 'rate': 4}, 'badge': {'discountRatio': 0, 'isNew': False}, 'metaData': {'title': 'Apple iPhone 11 64GB 4GB Ram 6.1 inç 12MP Akıllı Cep Telefonu Siyah Fiyatları ve Özellikleri', 'description': "Apple iPhone 11 64GB 4GB Ram 6.1 inç 12MP Akıllı Cep Telefonu Siyah modellerini tüm detayları ile Cimri.com'da inceleyin! Siz de Cimri.com ile Apple iPhone 11 64GB 4GB Ram 6.1 inç 12MP Akıllı Cep Telefonu Siyah fiyatlarına ve özelliklerine kolayca ulaşabilir, ürüne ait kullanıcı yorum ve değerlendirmelerini inceleyebilirsiniz. Dilerseniz fiyat ve özellik karşılaştıraması yapabilir ve son 1 yıla ait fiyat değişimlerini takip edebilirsiniz. Cimri.com ile siz de en ucuz Apple iPhone 11 64GB 4GB Ram 6.1 inç 12MP Akıllı Cep Telefonu Siyah fırsatlarını kaçırmayın!"}, 'specs': [{'name': 'Model Bilgisi', 'specEntries': [{'name': 'Iphone Modelleri', 'value': 'iPhone 11', 'order': 1}], 'order': 1}, {'name': 'Ekran Özellikleri', 'specEntries': [{'name': 'Ekran Boyutu', 'value': '6.1 inç', 'order': 3}, {'name': 'Ekran Teknolojisi', 'value': 'IPS LCD', 'order': 5}, {'name': 'Yan Ekran', 'value': 'Yok', 'order': 14}, {'name': 'Mavi Işık Filtresi', 'value': 'Var', 'order': 15}, {'name': 'Çözünürlük Standartı', 'value': 'HD+', 'order': 16}, {'name': 'Karanlık Modu', 'value': 'Var', 'order': 16}, {'name': 'Ekran Çözünürlüğü', 'value': '828x1792 Piksel', 'order': 21}, {'name': 'Ekran Gövde Oranı', 'value': '79 %', 'order': 21}, {'name': 'Piksel Yoğunluğu', 'value': '326 PPI', 'order': 22}, {'name': 'Multi Touch', 'value': 'Var', 'order': 22}, {'name': 'Dokunmatik Türü', 'value': 'Kapasitif Ekran', 'order': 23}, {'name': 'Ekran Parlaklığı (cd-m²)', 'value': '625', 'order': 26}, {'name': 'Çizilmeye Karşı Dayanıklılık', 'value': 'Var', 'order': 28}, {'name': 'Ekran Kontrast Oranı', 'value': '1400:1', 'order': 29}, {'name': 'Sürekli Açık Ekran', 'value': 'Yok', 'order': 30}], 'order': 1}, {'name': 'Teknik Özellikler', 'specEntries': [{'name': 'İşlemci Modeli', 'value': 'Apple A13 Bionic', 'order': 1}, {'name': 'İşlemci Frekansı', 'value': '2.66 GHz', 'order': 2}, {'name': 'RAM Kapasitesi', 'value': '4 GB RAM', 'order': 6}, {'name': 'İşletim Sistemi Versiyonu', 'value': 'iOS 13', 'order': 8}, {'name': 'İşletim Sistemi', 'value': 'iOS', 'order': 19}, {'name': 'İşlemci Çekirdek Sayısı', 'value': '6 Çekirdek', 'order': 26}, {'name': 'CPU Üretim Süreci', 'value': '7 nm', 'order': 27}, {'name': 'İşlemci Mimarisi', 'value': '64 bit', 'order': 28}, {'name': 'Ram Tipi', 'value': 'LPDDR4x', 'order': 30}, {'name': 'Ram Kanal Sayısı', 'value': 'Çift Kanal', 'order': 31}], 'order': 2}, {'name': 'Kamera Özellikleri', 'specEntries': [{'name': 'Kamera Çözünürlüğü', 'value': '12 MP', 'order': 4}, {'name': 'Ön (Selfie) Kamera Diyafram Açıklığı', 'value': 'F2.2', 'order': 6}, {'name': 'Ön (Selfie) Kamera Çözünürlüğü', 'value': '12 MP', 'order': 8}, {'name': 'Diyafram Açıklığı', 'value': 'F1.8', 'order': 9}, {'name': 'İkinci Arka Kamera', 'value': 'Var', 'order': 10}, {'name': 'İkinci Arka Kamera Diyafram Açıklığı', 'value': 'F2.4', 'order': 10}, {'name': 'Video Kayıt Çözünürlüğü', 'value': '2160p', 'order': 11}, {'name': 'Video FPS Değeri', 'value': '60 FPS', 'order': 12}, {'name': 'İkinci Arka Kamera Çözünürlüğü', 'value': '12 MP', 'order': 22}, {'name': 'Görüntü Sabitleyici', 'value': 'Var', 'order': 23}, {'name': 'Yüz Tanıma', 'value': 'Var', 'order': 24}, {'name': 'Slow Motion Video Çekimi', 'value': 'Var', 'order': 25}, {'name': 'Kamera Yapay Zeka (AI) Desteği', 'value': 'Yok', 'order': 26}, {'name': 'Time Lapse Video Çekimi', 'value': 'Var', 'order': 30}, {'name': 'Flaş Türü', 'value': 'Çift LED', 'order': 31}, {'name': 'Phase Detection', 'value': 'Var', 'order': 32}, {'name': 'HDR', 'value': 'Var', 'order': 33}, {'name': 'Zamanlayıcı', 'value': 'Var', 'order': 35}, {'name': 'Otomatik Odaklama', 'value': 'Var', 'order': 36}, {'name': 'Panoroma', 'value': 'Var', 'order': 37}, {'name': 'Coğrafi Konum Ekleme', 'value': 'Var', 'order': 38}], 'order': 3}, {'name': 'Tasarım', 'specEntries': [{'name': 'En', 'value': '75.7 mm', 'order': 1}, {'name': 'Boy', 'value': '150.9 mm', 'order': 2}, {'name': 'Kalınlık', 'value': '8.3 mm', 'order': 3}, {'name': 'Ağırlık', 'value': '194 gr', 'order': 7}, {'name': 'Gövde Malzemesi (Kapak)', 'value': 'Cam', 'order': 15}, {'name': 'Gövde Malzemesi (Çerçeve)', 'value': 'Alüminyum', 'order': 16}, {'name': 'Renk', 'value': 'Siyah', 'order': 31}], 'order': 4}, {'name': 'Batarya Özellikleri', 'specEntries': [{'name': 'Şarj Tipi', 'value': 'Lightning', 'order': 2}, {'name': 'Batarya Tipi', 'value': 'Li-ion', 'order': 11}, {'name': 'Kablosuz Hızlı Şarj', 'value': 'Yok', 'order': 12}, {'name': 'Kablosuz Hızlı Şarj Güç', 'value': 'Yok', 'order': 13}, {'name': 'Çıkarılabilir Batarya', 'value': 'Yok', 'order': 14}, {'name': 'Kablosuz Şarj', 'value': 'Var', 'order': 26}, {'name': 'Hızlı Şarj', 'value': 'Var', 'order': 27}], 'order': 5}, {'name': 'Depolama', 'specEntries': [{'name': 'Dahili Depolama (Hafıza)', 'value': '64 GB', 'order': 5}, {'name': 'Maksimum Hafıza Kartı Kapasitesi', 'value': 'Yok', 'order': 8}, {'name': 'Hafıza Kartı Desteği', 'value': 'Yok', 'order': 31}], 'order': 6}, {'name': 'Bağlantılar', 'specEntries': [{'name': 'Bluetooth Versiyonu', 'value': '5.0', 'order': 3}, {'name': 'Ses Çıkışı', 'value': 'Lightning', 'order': 5}, {'name': '4.5G Desteği', 'value': 'Var', 'order': 11}, {'name': 'Wİ-Fİ', 'value': 'Var', 'order': 11}, {'name': 'Wİ-Fİ Frekansı', 'value': 'Wi-Fi 6', 'order': 12}, {'name': 'Bluetooth', 'value': 'Var', 'order': 13}, {'name': 'Wİ-Fİ Hotspot', 'value': 'Var', 'order': 14}, {'name': 'Navigasyon', 'value': 'Var', 'order': 16}, {'name': '3G İndirme Hızı', 'value': '42.2 Mbps', 'order': 23}, {'name': '3G Yükleme Hızı', 'value': '5.76 Mbps', 'order': 24}, {'name': 'NFC', 'value': 'Var', 'order': 25}, {'name': '5G Desteği', 'value': 'Yok', 'order': 28}, {'name': 'Kızılötesi', 'value': 'Yok', 'order': 29}, {'name': 'USB Type-C', 'value': 'Yok', 'order': 30}], 'order': 7}, {'name': 'Ek Özellikler', 'specEntries': [{'name': 'Kalem', 'value': 'Yok', 'order': 11}, {'name': 'Güvenli Klasör', 'value': 'Yok', 'order': 12}, {'name': 'Antutu Puanı', 'value': '456900', 'order': 13}, {'name': 'Suya Dayanıklılık Seviyesi', 'value': 'IPX8', 'order': 14}, {'name': 'Toza Dayanıklılık Seviyesi', 'value': 'IP6X', 'order': 15}, {'name': 'Parmak İzi Okuyucu', 'value': 'Yok', 'order': 21}, {'name': 'Suya Dayanıklılık', 'value': 'Var', 'order': 23}, {'name': 'Toza Dayanıklılık', 'value': 'Var', 'order': 24}, {'name': 'Çıkış Tarihi', 'value': '2019', 'order': 35}, {'name': 'Parmak İzi Okuyucu Tipi', 'value': 'Yok', 'order': 37}, {'name': 'FM Radyo', 'value': 'Yok', 'order': 38}, {'name': 'Sim Kart Türü', 'value': 'Nano-SIM (4FF)', 'order': 39}], 'order': 8}, {'name': 'Sensörler', 'specEntries': [{'name': 'RGB Işık Sensörü', 'value': 'Yok', 'order': 2}, {'name': 'Bildirim Işığı', 'value': 'Yok', 'order': 8}, {'name': 'Barometre', 'value': 'Var', 'order': 31}, {'name': 'Jiroskop', 'value': 'Var', 'order': 32}, {'name': 'Hall Sensörü', 'value': 'Yok', 'order': 33}, {'name': 'Pusula', 'value': 'Var', 'order': 34}, {'name': 'Yakınlık Sensörü', 'value': 'Var', 'order': 35}, {'name': 'İvme Ölçer', 'value': 'Var', 'order': 36}], 'order': 9}, {'name': 'Kamera Özellikleri(Arka)', 'specEntries': [{'name': 'Elle Odaklama', 'value': 'Var', 'order': 8}, {'name': 'Seri Çekim Modu', 'value': 'Var', 'order': 10}, {'name': 'Zamanlayıcı', 'value': 'Var', 'order': 11}, {'name': 'Portre Modu', 'value': 'Var', 'order': 13}, {'name': 'Geniş Açı', 'value': 'Var', 'order': 13}, {'name': 'Yapay Zeka', 'value': 'Yok', 'order': 14}, {'name': 'Yüz Algılama', 'value': 'Var', 'order': 15}, {'name': 'Görüntü Sabitleyici', 'value': 'Var', 'order': 16}, {'name': 'Manuel Kontrol', 'value': 'Var', 'order': 17}, {'name': 'Hızlı Odaklama', 'value': 'Var', 'order': 52}, {'name': 'Odak Takibi', 'value': 'Var', 'order': 52}, {'name': 'Sesli Komut', 'value': 'Var', 'order': 52}, {'name': 'QR Kod Okuyucu', 'value': 'Yok', 'order': 55}], 'order': 10}, {'name': 'Kamera Özellikleri(Ön)', 'specEntries': [{'name': 'Geniş Açı', 'value': 'Yok', 'order': 2}, {'name': 'Otomatik Odaklama', 'value': 'Var', 'order': 3}, {'name': 'Panorama Özçekim', 'value': 'Yok', 'order': 4}, {'name': 'Seçmeli Odaklama', 'value': 'Yok', 'order': 5}, {'name': 'Sanal Flaş', 'value': 'Var', 'order': 8}, {'name': 'Zamanlayıcı', 'value': 'Var', 'order': 9}, {'name': 'Portre', 'value': 'Var', 'order': 18}, {'name': 'Sesle Komut', 'value': 'Yok', 'order': 19}, {'name': 'Yapay Zeka', 'value': 'Yok', 'order': 20}], 'order': 11}, {'name': 'Kamera Özellikleri(Video)', 'specEntries': [{'name': 'FPS', 'value': '60', 'order': 1}, {'name': 'Çözünürlük Standardı', 'value': '2160p', 'order': 2}, {'name': 'HDR', 'value': 'Var', 'order': 33}, {'name': 'Portre', 'value': 'Yok', 'order': 52}, {'name': 'Görüntü Sabitleme', 'value': 'Var', 'order': 53}, {'name': 'Ağır Çekim', 'value': 'Var', 'order': 54}, {'name': 'Hızlı Çekim', 'value': 'Var', 'order': 55}], 'order': 12}, {'name': 'USB', 'specEntries': [{'name': 'OTG', 'value': 'Var', 'order': 38}, {'name': 'DisplayPort', 'value': 'Yok', 'order': 39}, {'name': 'E-Sim', 'value': 'Var', 'order': 40}, {'name': 'Micro USB', 'value': 'Yok', 'order': 50}, {'name': 'Lightning', 'value': 'Var', 'order': 52}], 'order': 13}], 'topSpecs': [{'name': 'Dahili Depolama (Hafıza)', 'value': '64 GB', 'order': 5}, {'name': 'Diyafram Açıklığı', 'value': 'F1.8', 'order': 9}, {'name': 'Ekran Boyutu', 'value': '6.1 inç', 'order': 3}, {'name': 'Kamera Çözünürlüğü', 'value': '12 MP', 'order': 4}, {'name': 'Ön (Selfie) Kamera Çözünürlüğü', 'value': '12 MP', 'order': 8}], 'unitType': None, 'variantsHeader': 'Renk', 'variants': [{'id': '337337465', 'imageId': 178955343, 'minPrice': 13229, 'shortTitle': 'Beyaz', 'title': 'Apple iPhone 11 64GB 4GB Ram 6.1 inç 12MP Akıllı Cep Telefonu Beyaz', 'url': '/cep-telefonlari/en-ucuz-apple-iphone-11-64gb-4gb-ram-6-1-inc-12mp-akilli-cep-telefonu-beyaz-fiyatlari,337337465', 'unitPrice': None}, {'id': '337339455', 'imageId': 178955504, 'minPrice': 13249, 'shortTitle': 'Sarı', 'title': 'Apple iPhone 11 64GB 4GB Ram 6.1 inç 12MP Akıllı Cep Telefonu Sarı', 'url': '/cep-telefonlari/en-ucuz-apple-iphone-11-64gb-4gb-ram-6-1-inc-12mp-akilli-cep-telefonu-sari-fiyatlari,337339455', 'unitPrice': None}, {'id': '337338130', 'imageId': 178955431, 'minPrice': 13299, 'shortTitle': 'Yeşil', 'title': 'Apple iPhone 11 64GB 4GB Ram 6.1 inç 12MP Akıllı Cep Telefonu Yeşil', 'url': '/cep-telefonlari/en-ucuz-apple-iphone-11-64gb-4gb-ram-6-1-inc-12mp-akilli-cep-telefonu-yesil-fiyatlari,337338130', 'unitPrice': None}, {'id': '337338937', 'imageId': 178955480, 'minPrice': 13299, 'shortTitle': 'Mor', 'title': 'Apple iPhone 11 64GB 4GB Ram 6.1 inç 12MP Akıllı Cep Telefonu Mor', 'url': '/cep-telefonlari/en-ucuz-apple-iphone-11-64gb-4gb-ram-6-1-inc-12mp-akilli-cep-telefonu-mor-fiyatlari,337338937', 'unitPrice': None}, {'id': '337340793', 'imageId': 219571586, 'minPrice': 12986, 'shortTitle': 'Siyah', 'title': 'Apple iPhone 11 64GB 4GB Ram 6.1 inç 12MP Akıllı Cep Telefonu Siyah', 'url': '/cep-telefonlari/en-ucuz-apple-iphone-11-64gb-4gb-ram-6-1-inc-12mp-akilli-cep-telefonu-siyah-fiyatlari,337340793', 'unitPrice': None}, {'id': '337340093', 'imageId': 178955538, 'minPrice': 13385.59, 'shortTitle': 'Kırmızı', 'title': 'Apple iPhone 11 64GB 4GB Ram 6.1 inç 12MP Akıllı Cep Telefonu Kırmızı', 'url': '/cep-telefonlari/en-ucuz-apple-iphone-11-64gb-4gb-ram-6-1-inc-12mp-akilli-cep-telefonu-kirmizi-fiyatlari,337340093', 'unitPrice': None}], 'mainVariantsHeader': 'Kapasite', 'mainVariants': [{'id': '337341638', 'imageId': 178955480, 'minPrice': 12986, 'shortTitle': '64 GB', 'title': 'Apple iPhone 11 64GB 4GB Ram 6.1 inç 12MP Akıllı Cep Telefonu', 'url': '/cep-telefonlari/en-ucuz-apple-iphone-11-64gb-4gb-ram-6-1-inc-12mp-akilli-cep-telefonu-fiyatlari,a337341638'}, {'id': '331840845', 'imageId': 178954771, 'minPrice': 13999, 'shortTitle': '128 GB', 'title': 'Apple iPhone 11 128GB Akıllı Cep Telefonu', 'url': '/cep-telefonlari/en-ucuz-apple-iphone-11-128gb-akilli-cep-telefonu-fiyatlari,a331840845'}, {'id': '337357780', 'imageId': 178954858, 'minPrice': 20899, 'shortTitle': '256 GB', 'title': 'Apple iPhone 11 256GB 4GB Ram 6.1 inç 12MP Akıllı Cep Telefonu', 'url': '/cep-telefonlari/en-ucuz-apple-iphone-11-256gb-4gb-ram-6-1-inc-12mp-akilli-cep-telefonu-fiyatlari,a337357780'}], 'brand': {'id': 4, 'name': 'Apple', 'slug': 'apple'}, 'category': {'id': '130', 'name': 'Cep Telefonları', 'slug': 'cep-telefonlari'}}
I'm creating a python script to look at web scrapping. I can extract most the text I need within div or class but am having an issue with tag.
url1 = "https://www.wowhead.com/"
soup1 = BeautifulSoup(html1, "html.parser")
test = soup1.find_all('div', attrs={"featured-content-block type-today-in-wow today-in-wow"})[0].find_all('script')[-1]
this returns:
<script>new WH.News.TodayInWoW([{"assaults":{"duration":302400,"expansion":8,...."zone":10288}}}]);</script>
What I need is in the WH.News.TodayInWoW function (not sure what part of script to call it, emboldened) but I don't know how to extract this. Either a list or dictionary, I plan to filter out the keys/values I want.
Anything would be appreciated, I have already looked at a few other BS and email extraction but doesnt work for me
You can try with the substring (notice the [31: -11] at the end):
test = soup1.find_all('div', attrs={"featured-content-block type-today-in-wow today-in-wow"})[0].find_all('script')[-1][31: -11]
To get the actual dictionary, eval the string:
d = eval(test)
If I'm understanding this correctly you cannot web-scrape a function. What you are doing is just web scraping the parameters of the function. Assuming even if the website has made the script public, you'd need to go to inspect element > sources > script and then try to find the js file where that function is!
You could use BeautifulSoup to extract as you did, then you would need to do some string/data manipulation and the json package to read that into python as a list and dictionary. Or simply, what I did was use regex to extract the json from the html request. Not sure what you are after exactly, but I converted that into a table with pandas to show you the output.
import requests
import re
import json
url1 = "https://www.wowhead.com/"
pattern = 'WH.News.TodayInWoW\((\[.*\])'
html1 = requests.get(url1).text
jsonStr = re.search(pattern, html1).group(1)
jsonData = json.loads(jsonStr)
Output:
print(jsonData)
[{'assaults': [{'duration': 302400, 'expansion': 8, 'iconActive': 'ui_sigil_kyrian', 'iconInactive': 'item_rep_deathsadvance_01', 'id': 'covenant-assault', 'label': 'Covenant Assault', 'name': 'Kyrian Assault', 'requiresExpansion': False, 'type': 'covenant-assault', 'upcoming': [1644634800, 1644937200, 1645239600, 1645542000, 1645844400], 'url': '/guides/covenant-assaults-maw-invasions'}, {'duration': 21600, 'expansion': 6, 'iconActive': 'inv_misc_map08', 'iconInactive': 'achievement_faction_legionfall', 'id': 'legion-assaults', 'label': 'Legion Assaults', 'name': None, 'requiresExpansion': True, 'type': 'legion-assaults', 'upcoming': [1644984000, 1645050600, 1645117200, 1645183800, 1645250400], 'url': '/legion-assaults-guide'}, {'duration': 25200, 'expansion': 7, 'iconActive': 'inv_misc_map08', 'iconInactive': 'inv_misc_map08', 'id': 'faction-assaults', 'label': 'Faction Assaults', 'name': None, 'requiresExpansion': True, 'type': 'faction-assaults', 'upcoming': [1644955200, 1645023600, 1645092000, 1645160400, 1645228800], 'url': '/guides/battle-for-azeroth-incursions'}, {'duration': 604800, 'expansion': 7, 'iconActive': 'achievement_zone_uldum', 'iconInactive': 'inv_eyeofnzothpet', 'id': 'nzoth-assaults-major', 'label': "N'Zoth Assaults - Major", 'name': 'Uldum', 'requiresExpansion': True, 'type': 'nzoth-assaults-major', 'upcoming': [1644332400, 1644937200, 1645542000, 1646146800, 1646751600], 'url': '/guides/visions-of-nzoth-assaults'}, {'duration': 302400, 'expansion': 7, 'iconActive': 'achievement_zone_valeofeternalblossoms', 'iconInactive': 'inv_eyeofnzothpet', 'id': 'nzoth-assaults-minor', 'label': "N'Zoth Assaults - Minor", 'name': 'Vale of Eternal Blossoms', 'requiresExpansion': True, 'type': 'nzoth-assaults-minor', 'upcoming': [1644634800, 1644937200, 1645239600, 1645542000, 1645844400], 'url': '/guides/visions-of-nzoth-assaults'}], 'id': 'US', 'name': 'NA', 'progression': {'defeatedBosses': 10, 'icon': 'achievement_raid_torghastraid', 'id': 'progression', 'name': 'Sanctum of Domination', 'topGuildCount': 1683, 'totalBosses': 10, 'type': 'progression', 'url': 'https://www.wowhead.com/guides/sanctum-of-domination-raid-overview-strategy-boss-guides-rewards'}, 'warfronts': {'9734': {'expansion': 7, 'contributions': None, 'icon': 'achievement_zone_arathihighlands_01', 'name': 'Arathi Highlands', 'progress': 17.5049603, 'requiresExpansion': True, 'side': 2, 'stateId': 'attacking', 'stateName': 'Horde Attacking', 'url': '/how-to-win-battle-for-stromgarde-warfront-strategy-guide', 'zone': 9734}, '10288': {'expansion': 7, 'contributions': None, 'icon': 'achievement_zone_darkshore_01', 'name': 'Darkshore', 'progress': 86.73307299613953, 'requiresExpansion': True, 'side': 1, 'stateId': 'contributing', 'stateName': 'Alliance Contributing', 'url': '/guides/launching-a-warfront-through-contributions', 'zone': 10288}}}, {'assaults': [{'duration': 302400, 'expansion': 8, 'iconActive': 'ui_sigil_kyrian', 'iconInactive': 'item_rep_deathsadvance_01', 'id': 'covenant-assault', 'label': 'Covenant Assault', 'name': 'Kyrian Assault', 'requiresExpansion': False, 'type': 'covenant-assault', 'upcoming': [1644692400, 1644994800, 1645297200, 1645599600, 1645902000], 'url': '/guides/covenant-assaults-maw-invasions'}, {'duration': 21600, 'expansion': 6, 'iconActive': 'inv_misc_map08', 'iconInactive': 'achievement_faction_legionfall', 'id': 'legion-assaults', 'label': 'Legion Assaults', 'name': None, 'requiresExpansion': True, 'type': 'legion-assaults', 'upcoming': [1644955200, 1645021800, 1645088400, 1645155000, 1645221600], 'url': '/legion-assaults-guide'}, {'duration': 25200, 'expansion': 7, 'iconActive': 'inv_tiragardesound', 'iconInactive': 'inv_misc_map08', 'id': 'faction-assaults', 'label': 'Faction Assaults', 'name': 'Tiragarde Sound', 'requiresExpansion': True, 'type': 'faction-assaults', 'upcoming': [1644922800, 1644991200, 1645059600, 1645128000, 1645196400], 'url': '/guides/battle-for-azeroth-incursions'}, {'duration': 604800, 'expansion': 7, 'iconActive': 'achievement_zone_uldum', 'iconInactive': 'inv_eyeofnzothpet', 'id': 'nzoth-assaults-major', 'label': "N'Zoth Assaults - Major", 'name': 'Uldum', 'requiresExpansion': True, 'type': 'nzoth-assaults-major', 'upcoming': [1644390000, 1644994800, 1645599600, 1646204400, 1646809200], 'url': '/guides/visions-of-nzoth-assaults'}, {'duration': 302400, 'expansion': 7, 'iconActive': 'achievement_zone_valeofeternalblossoms', 'iconInactive': 'inv_eyeofnzothpet', 'id': 'nzoth-assaults-minor', 'label': "N'Zoth Assaults - Minor", 'name': 'Vale of Eternal Blossoms', 'requiresExpansion': True, 'type': 'nzoth-assaults-minor', 'upcoming': [1644692400, 1644994800, 1645297200, 1645599600, 1645902000], 'url': '/guides/visions-of-nzoth-assaults'}], 'id': 'EU', 'name': 'EU', 'progression': {'defeatedBosses': 10, 'icon': 'achievement_raid_torghastraid', 'id': 'progression', 'name': 'Sanctum of Domination', 'topGuildCount': 1683, 'totalBosses': 10, 'type': 'progression', 'url': 'https://www.wowhead.com/guides/sanctum-of-domination-raid-overview-strategy-boss-guides-rewards'}, 'warfronts': {'9734': {'expansion': 7, 'contributions': None, 'icon': 'achievement_zone_arathihighlands_01', 'name': 'Arathi Highlands', 'progress': 0.9389017708599567, 'requiresExpansion': True, 'side': 2, 'stateId': 'contributing', 'stateName': 'Horde Contributing', 'url': '/guides/launching-a-warfront-through-contributions', 'zone': 9734}, '10288': {'expansion': 7, 'contributions': None, 'icon': 'achievement_zone_darkshore_01', 'name': 'Darkshore', 'progress': 49.4943783, 'requiresExpansion': True, 'side': 2, 'stateId': 'attacking', 'stateName': 'Horde Attacking', 'url': '/guides/rewards-from-the-darkshore-warfront', 'zone': 10288}}}]
Into Dataframe:
import pandas as pd
df = pd.json_normalize(jsonData, record_path=['assaults'])
Output:
print(df.to_string())
duration expansion iconActive iconInactive id label name requiresExpansion type upcoming url
0 302400 8 ui_sigil_kyrian item_rep_deathsadvance_01 covenant-assault Covenant Assault Kyrian Assault False covenant-assault [1644634800, 1644937200, 1645239600, 1645542000, 1645844400] /guides/covenant-assaults-maw-invasions
1 21600 6 inv_misc_map08 achievement_faction_legionfall legion-assaults Legion Assaults None True legion-assaults [1644984000, 1645050600, 1645117200, 1645183800, 1645250400] /legion-assaults-guide
2 25200 7 inv_misc_map08 inv_misc_map08 faction-assaults Faction Assaults None True faction-assaults [1644955200, 1645023600, 1645092000, 1645160400, 1645228800] /guides/battle-for-azeroth-incursions
3 604800 7 achievement_zone_uldum inv_eyeofnzothpet nzoth-assaults-major N'Zoth Assaults - Major Uldum True nzoth-assaults-major [1644332400, 1644937200, 1645542000, 1646146800, 1646751600] /guides/visions-of-nzoth-assaults
4 302400 7 achievement_zone_valeofeternalblossoms inv_eyeofnzothpet nzoth-assaults-minor N'Zoth Assaults - Minor Vale of Eternal Blossoms True nzoth-assaults-minor [1644634800, 1644937200, 1645239600, 1645542000, 1645844400] /guides/visions-of-nzoth-assaults
5 302400 8 ui_sigil_kyrian item_rep_deathsadvance_01 covenant-assault Covenant Assault Kyrian Assault False covenant-assault [1644692400, 1644994800, 1645297200, 1645599600, 1645902000] /guides/covenant-assaults-maw-invasions
6 21600 6 inv_misc_map08 achievement_faction_legionfall legion-assaults Legion Assaults None True legion-assaults [1644955200, 1645021800, 1645088400, 1645155000, 1645221600] /legion-assaults-guide
7 25200 7 inv_tiragardesound inv_misc_map08 faction-assaults Faction Assaults Tiragarde Sound True faction-assaults [1644922800, 1644991200, 1645059600, 1645128000, 1645196400] /guides/battle-for-azeroth-incursions
8 604800 7 achievement_zone_uldum inv_eyeofnzothpet nzoth-assaults-major N'Zoth Assaults - Major Uldum True nzoth-assaults-major [1644390000, 1644994800, 1645599600, 1646204400, 1646809200] /guides/visions-of-nzoth-assaults
9 302400 7 achievement_zone_valeofeternalblossoms inv_eyeofnzothpet nzoth-assaults-minor N'Zoth Assaults - Minor Vale of Eternal Blossoms True nzoth-assaults-minor [1644692400, 1644994800, 1645297200, 1645599600, 1645902000] /guides/visions-of-nzoth-assaults
I have a JSON object that looks like this:
data = {'A': {'code': 'Ok',
'tracepoints': [None,
None,
{'alternatives_count': 0,
'location': [-122.419189, 37.753805],
'distance': 28.078003,
'hint': '5Qg7hUqpFQA2AAAAOgAAAAwAAAAPAAAAiVMWQq2VIEIAuABB7FgoQTYAAAA6AAAADAAAAA8AAAD4RAAACwi0-M0TQALvB7T4yRRAAgEAXwX5Wu6N',
'name': '23rd Street',
'matchings_index': 0,
'waypoint_index': 0},
{'alternatives_count': 0,
'location': [-122.417866, 37.75389],
'distance': 26.825184,
'hint': 'K8w6BRinFYAdAAAACwAAAA0AAAAAAAAAIxmmQTSs6kCiuRFBAAAAAB0AAAALAAAADQAAAAAAAAD4RAAANg20-CIUQAJNDbT4MRNAAgIAnxD5Wu6N',
'name': '23rd Street',
'matchings_index': 0,
'waypoint_index': 1},
{'alternatives_count': 0,
'location': [-122.416896, 37.75395],
'distance': 16.583412,
'hint': 'Jcw6BSzMOoUqAAAAQwAAABAAAAANAAAA0i_uQb3SOEKKPC9BG1EaQSoAAABDAAAAEAAAAA0AAAD4RAAAABG0-F4UQALyELT48xRAAgEAnxD5Wu6N',
'name': '23rd Street',
'matchings_index': 0,
'waypoint_index': 2},
{'alternatives_count': 7,
'location': [-122.415502, 37.754028],
'distance': 10.013916,
'hint': 'Jsw6hbN6kQBmAAAACAAAABAAAAANAAAAQOKOQg89nkCKPC9BEMcOQWYAAAAIAAAAEAAAAA0AAAD4RAAAcha0-KwUQAJ6FrT4UhRAAgEAbwX5Wu6N',
'name': '23rd Street',
'matchings_index': 0,
'waypoint_index': 3}],
'matchings': [{'duration': 50.6,
'distance': 325.2,
'weight': 50.6,
'geometry': 'y{h_gAh~znhF}#k[OmFMoFcAea#IeD[uMAYKsDMsDAe#}#u_#g#aTMwFMwFwAqq#',
'confidence': 0.374625,
'weight_name': 'routability',
'legs': [{'steps': [],
'weight': 18.8,
'distance': 116.7,
'annotation': {'nodes': [1974590926,
4763953263,
65359046,
4763953265,
5443374298,
2007343352]},
'summary': '',
'duration': 18.8},
{'steps': [],
'weight': 12.2,
'distance': 85.6,
'annotation': {'nodes': [5443374298,
2007343352,
4763953266,
65359043,
4763953269,
2007343354,
4763953270]},
'summary': '',
'duration': 12.2},
{'steps': [],
'weight': 19.6,
'distance': 122.9,
'annotation': {'nodes': [2007343354,
4763953270,
65334199,
4763953274,
2007343347]},
'summary': '',
'duration': 19.6}]}]},
'B': {'code': 'Ok',
'tracepoints': [{'alternatives_count': 0,
'location': [-122.387971, 37.727587],
'distance': 11.53267,
'hint': 'xHWRAEJ2kYALAAAArQAAAA4AAAAsAAAAnpH1QDVG8EJWgBdBa2v0QQsAAACtAAAADgAAACwAAAD4RAAA_YG0-GOtPwJKgrT4t60_AgIA3wf5Wu6N',
'name': 'Underwood Avenue',
'matchings_index': 0,
'waypoint_index': 0},
{'alternatives_count': 0,
'location': [-122.388563, 37.727175],
'distance': 13.565054,
'hint': 'w3WRgBuxOgVPAAAACAAAABMAAAASAAAA7ONaQo4CrUDv7U1BJdFAQU8AAAAIAAAAEwAAABIAAAD4RAAArX-0-MerPwIsgLT4gqs_AgIAbw35Wu6N',
'name': 'Jennings Street',
'matchings_index': 0,
'waypoint_index': 1},
{'alternatives_count': 1,
'location': [-122.388478, 37.725984],
'distance': 9.601917,
'hint': 't3WRABexOoWcAAAAbAAAABEAAAALAAAAdujYQqu4lUJXHD1B9-ruQJwAAABsAAAAEQAAAAsAAAD4RAAAAoC0-CCnPwJCgLT4Zqc_AgIAHxP5Wu6N',
'name': 'Wallace Avenue',
'matchings_index': 0,
'waypoint_index': 2}],
'matchings': [{'duration': 50,
'distance': 270.4,
'weight': 50,
'geometry': 'euu}fAd_~lhFoAlCMTuAvCvC|Bh#`#hXbUnAdADBhDzCzClCXVzZnW\\X~CnC~#qBLWnWej#',
'confidence': 1e-06,
'weight_name': 'routability',
'legs': [{'steps': [],
'weight': 17.8,
'distance': 84.8,
'annotation': {'nodes': [5443147626,
6360865540,
6360865536,
65307580,
6360865535,
6360865539,
6360865531]},
'summary': '',
'duration': 17.8},
{'steps': [],
'weight': 32.2,
'distance': 185.6,
'annotation': {'nodes': [6360865539,
6360865531,
6360865525,
65343521,
6360865527,
6360865529,
6360865523,
6360865520,
65321110,
6360865519,
6360865522,
6376329343]},
'summary': '',
'duration': 32.2}]}]},
'C': {'code': 'Ok',
'tracepoints': [None,
None,
{'alternatives_count': 0,
'location': [-122.443682, 37.713254],
'distance': 6.968076,
'hint': 'QXo6hUR6OgUAAAAANQAAAAAAAAAkAAAAAAAAAOCMMUEAAAAA_Z1yQQAAAAAbAAAAAAAAACQAAAD4RAAAXqiz-GZ1PwKiqLP4hnU_AgAAzxL5Wu6N',
'name': '',
'matchings_index': 0,
'waypoint_index': 0},
{'alternatives_count': 0,
'location': [-122.442428, 37.714335],
'distance': 16.488956,
'hint': 'E3o6BVRukYAJAAAAIgAAAGgAAAAUAAAA2RnSQL_5uUEPjI9CBTlaQQkAAAAiAAAAaAAAABQAAAD4RAAARK2z-J95PwKTrLP4b3k_AgEAXxX5Wu6N',
'name': 'Allison Street',
'matchings_index': 0,
'waypoint_index': 1},
{'alternatives_count': 1,
'location': [-122.441751, 37.712761],
'distance': 17.311636,
'hint': 'Fno6hRl6OgWZAAAANwAAAAAAAAAKAAAAH4vUQgKXFkIAAAAAXtbYQJkAAAA3AAAAAAAAAAoAAAD4RAAA6a-z-HlzPwKjsLP4q3M_AgAAHwr5Wu6N',
'name': 'Allison Street',
'matchings_index': 0,
'waypoint_index': 2}],
'matchings': [{'duration': 64.1,
'distance': 420.1,
'weight': 66.7,
'geometry': 'kuy|fAbyjphFcBxEmE`FqJkKiBqBuP}Qgc#ie#eAiAcB}ArA_Eb#mAjKkDnBo#fe#mOrw#kW',
'confidence': 7.3e-05,
'weight_name': 'routability',
'legs': [{'steps': [],
'weight': 40.1,
'distance': 235.2,
'annotation': {'nodes': [5440513673,
5440513674,
5440513675,
65363070,
1229920760,
65307726,
6906452420,
1229920717,
65361047,
1229920749,
554163599,
3978809925]},
'summary': '',
'duration': 37.5},
{'steps': [],
'weight': 26.6,
'distance': 184.9,
'annotation': {'nodes': [554163599, 3978809925, 65345518, 8256268328]},
'summary': '',
'duration': 26.6}]}]}}
I would like to extract the values under the key nodes per user (A, B and C) and store these values in a pandas dataframe, together with the corresponding user. Like below:
value user
1974590926 A
4763953263 A
65359046 A
4763953265 A
5443374298 A
2007343352 A
5443374298 A
2007343352 A
4763953266 A
65359043 A
4763953269 A
2007343354 A
4763953270 A
2007343354 A
4763953270 A
65334199 A
4763953274 A
2007343347 A
5443147626 B
6360865540 B
6360865536 B
65307580 B
6360865535 B
6360865539 B
6360865531 B
6360865539 B
6360865531 B
6360865525 B
65343521 B
6360865527 B
6360865529 B
6360865523 B
6360865520 B
65321110 B
6360865519 B
6360865522 B
6376329343 B
5440513673 C
5440513674 C
5440513675 C
65363070 C
1229920760 C
65307726 C
6906452420 C
1229920717 C
65361047 C
1229920749 C
554163599 C
3978809925 C
554163599 C
3978809925 C
65345518 C
8256268328 C
I am able to extract and store only the nodes belonging to user C to a pandas dataframe with the code below. However, I struggle to add the user column and the other nodes with their corresponding user. Any ideas?
import pandas as pd
nodes_df = pd.DataFrame({'node':{}})
for user in output[user]['matchings'][0]['legs']:
result = user['annotation']['nodes']
values_temp = pd.DataFrame(result, columns=['value'])
values_df = values_df.append(values_temp, ignore_index=True)
values_df.node = values_df.value.astype(int)
values_df
value
0 5440513673
1 5440513674
2 5440513675
3 65363070
4 1229920760
5 65307726
6 6906452420
7 1229920717
8 65361047
9 1229920749
10 554163599
11 3978809925
12 554163599
13 3978809925
14 65345518
15 8256268328
You can use json_normalize() with record_path and then concat() the users:
dfs = []
for user in output.keys():
df = pd.json_normalize(output, record_path=[user, 'matchings', 'legs', 'annotation', 'nodes'])
df['user'] = user
dfs.append(df)
nodes_df = pd.concat(dfs).rename(columns={0: 'node'})
# node user
# 1974590926 A
# 4763953263 A
# 65359046 A
# ... ...
# 3978809925 C
# 65345518 C
# 8256268328 C
If there are some users with missing matchings, you can check if 'matchings' in output[user]:
dfs = []
for user in output.keys():
if 'matchings' in output[user]:
df = pd.json_normalize(output, record_path=[user, 'matchings', 'legs', 'annotation', 'nodes'])
df['user'] = user
dfs.append(df)
nodes_df = pd.concat(dfs).rename(columns={0: 'node'})
If the output keys are like ('2018-02-03', 'A') and you're iterating them as trip, you need to access its date and user as trip[0] and trip[1]:
dfs = []
for trip in output.keys():
if 'matchings' in output[trip]:
df = pd.json_normalize(output, record_path=[trip, 'matchings', 'legs', 'annotation', 'nodes'])
df['date'] = trip[0]
df['user'] = trip[1]
dfs.append(df)
nodes_df = pd.concat(dfs).rename(columns={0: 'node'})
We want to put all the node values in [legs]
If you want the simplest way with just for loop:
nodes = []
user = []
for i in output.keys():
for j in output[i]['matchings'][0]['legs']:
for k in j['annotation']['nodes']:
col1.append(k)
col2.append(i)
d = {'nodes':nodes, 'user':user}
df = pd.DataFrame(data=d)
print(df)
You could use the jmespath module to extract the data, before recombining within the dataframe; you should get some speed up, since the iteration is within the dictionary:
The summary for jmespath is : if accessing a key, then use dot, if the data is within a list, then use the [] to access the data:
#pip install jmespath
import jmespath
from itertools import chain
query ={letter: jmespath.compile(f"{letter}.matchings[].legs[].annotation.nodes")
for letter in ("A", "B", "C")}
result = {letter: pd.DataFrame(chain.from_iterable(expression.search(output)),
columns=['node'])
for letter, expression in query.items()}
result = pd.concat(result).droplevel(-1).rename_axis(index='user').reset_index()
result.head(15)
user node
0 A 1974590926
1 A 4763953263
2 A 65359046
3 A 4763953265
4 A 5443374298
5 A 2007343352
6 A 5443374298
7 A 2007343352
8 A 4763953266
9 A 65359043
10 A 4763953269
11 A 2007343354
12 A 4763953270
13 A 2007343354
14 A 4763953270