I am trying to store different data for each server I'm setting my bot for I have looked through many forums and haven't found anything that fits my need, Here is my code:
#client.command()
async def pop(ctx, arg):
global pop2
pop2 = arg
URL = 'https://www.battlemetrics.com/servers/rust/' + arg
page = requests.get(URL)
soup = BeautifulSoup(page.content, 'html.parser')
title = soup.find("h2").get_text()
title = title.replace('Connect', '')
soup = BeautifulSoup(page.content, 'html.parser')
page = soup.find('dl', class_='css-1i1egz4')
pop = page.find("dt", text="Player count")
status = page.find("dt", text="Status")
status1 = status.findNext("dd").get_text()
pop1 = pop.findNext("dd").get_text()
if status1 == "offline":
colour = 0xff0000
elif status1 == "online":
colour = 0x33ff0a
elif status1 == "dead":
colour = 0xff0000
embed = discord.Embed(title=title, description=status1, color=colour)
embed.add_field(name="Server", value=URL, inline=False)
embed.add_field(name="pop", value=pop1, inline=False)
await ctx.send("The server has been set to:")
await ctx.send(embed=embed)
What are you trying to achieve? just storing per-guild data? you could json on a small scale, or look into larger databases like MySQL or MongoDB
Related
I want to improve my code, changing the synchronous functions to asynchronous for a faster data extraction speed, but every time I run the program, I get the Print("Error").
async def soup(html):
soup = BeautifulSoup(html, 'html.parser')
return soup
async def title_bs4(html, tag, classes):
soup = await soup(html)
title = soup.findAll(tag, attrs={"class": classes})
title = [i.text for i in title]
return title
async def url_bs4(html, tag, classes):
soup = await soup(html)
url = soup.findAll(tag, attrs={"class": classes})
url = [i.text for i in url]
return url
async def price_xpath(html):
soup = await soup(html)
dom = etree.HTML(str(soup))
price = dom.xpath('//li[#class="ui-search-layout__item shops__layout-item"]//div[#class="ui-search-result__content-columns shops__content-columns"]/div[#class="ui-search-result__content-column ui-search-result__content-column--left shops__content-columns-left"]/div[1]/div//div[#class="ui-search-price__second-line shops__price-second-line"]//span[#class="price-tag-amount"]/span[2]')
price = [i.text.replace('.', '') for i in price]
return price
async def page_number_bs4(html, tag, classes):
soup = await soup(html)
page_number = soup.find(tag, attrs={"class": classes}).text
page_number = int(page_number)
return page_number
async def number_of_pages_bs4(html, tag, classes):
soup = await soup(html)
number_of_pages = soup.find(tag, attrs={"class": classes}).text
number_of_pages = int(number_of_pages.split(" ")[1])
return number_of_pages
async def next_xpath(html):
soup = await soup(html)
dom = etree.HTML(str(soup))
next = dom.xpath(
'//div[#class="ui-search-pagination shops__pagination-content"]/ul/li[contains(#class,"--next")]/a')[0].get('href')
return next
async def main(product):
web = "Mercado libre"
list_titles = []
list_urls = []
list_prices = []
next = 'https://listado.mercadolibre.com.co/' + str(product)
async with aiohttp.ClientSession() as session:
async with session.get(next) as response:
while True:
try:
title = await title_bs4(response, 'h2', 'ui-search-item__title shops__item-title')
list_titles.extend(title)
url = await url_bs4(response, 'a', 'ui-search-item__group__element shops__items-group-details ui-search-link')
list_titles.extend(url)
price = await price_xpath(response)
list_titles.extend(price)
page_number = await page_number_bs4(response, 'span', 'andes-pagination__link')
number_of_pages = await number_of_pages_bs4(response, 'li', 'andes-pagination__page-count')
except:
print("Error")
break
if page_number == number_of_pages:
break
next = await next_xpath(response)
df = pd.DataFrame({"shop": web, "titles": list_titles,
"links": list_urls, "prices": list_prices})
df.prices = df.prices.map(
lambda x: float(re.search(r"\d+", x).group(0)))
df.to_json("templates/product.json", orient='records')
return df
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
asyncio.run(main('samsung'))
except KeyboardInterrupt:
pass
My synchronous functions work very well but are very slow when it comes to wanting to extract data from the paginations.
I want to find the solution when running the program or if there is another better alternative to what I am looking for above.
I scraped the data using Playwright by css selectors. But when i tried to scrape the data using Xpath by 'page.evaluate' method, all the data became 'Not Available'. Is there any method to scrape the data by XPath selectors in Playwright.
Using Playwright by css selectors.
import asyncio
import pandas as pd
from playwright.async_api import async_playwright
# Extract the Product links
async def get_product_links(page):
all_items = await page.query_selector_all('.a-link-normal.DealCardDynamic-module__linkOutlineOffset_2XU8RDGmNg2HG1E-ESseNq')
product_links = []
for item in all_items:
link = await item.get_attribute('href')
product_links.append(link)
return product_links
# Extract the Product name
async def get_product_name(page):
try:
product_name = await (await page.query_selector("#productTitle")).text_content()
except:
product_name = "Not Available"
return product_name
async def main():
async with async_playwright() as pw:
browser = await pw.chromium.launch()
page = await browser.new_page()
await page.goto('https://www.amazon.in/gp/goldbox?deals-widget=%257B%2522version%2522%253A1%252C%2522viewIndex%2522%253A0%252C%2522presetId%2522%253A%252215C82F45284EDD496F94A2C368D1B4BD%2522%252C%2522sorting%2522%253A%2522BY_SCORE%2522%257D')
product_links = await get_product_links(page)
data = []
# Get the product data
for link in product_links:
await perform_request_with_retry(page, link)
product_name = await get_product_name(page)
data.append((link, product_name))
# Save the extracted data to a dataframe
df = pd.DataFrame(data, columns=['Product Link', 'Product Name'])
# Save the extracted data to a csv file
df.to_csv('product_details.csv', index=False)
print('CSV file has been written successfully.')
await browser.close()
# Execute the scraping and saving of Amazon today's deals - musical instruments
if __name__ == '__main__':
asyncio.run(main())
Using Xpath by page.evaluate method
async def get_product_name(page):
try:
product_name = await page.evaluate("//span[#id='productTitle']/text()")
except:
product_name = "Not Available"
return product_name
Sample Link :
https://www.amazon.in/gp/goldbox?deals-widget=%257B%2522version%2522%253A1%252C%2522viewIndex%2522%253A0%252C%2522presetId%2522%253A%252215C82F45284EDD496F94A2C368D1B4BD%2522%252C%2522sorting%2522%253A%2522BY_SCORE%2522%257D
Is there any method to scrape the data by XPath selectors in Playwright?
I'm making a telegram bot that will send SMS through a parser and I need to loop until about 20 SMS are sent.
I am using the telebot library to create a bot and for the parser I have requests and BeautifulSoup.
import telebot
import requests
from bs4 import BeautifulSoup
from telebot import types
bot = telebot.TeleBot('my token')
#bot.message_handler(commands=['start'])
def start(message):
mess = f'Привет, сегодня у меня для тебя 100 Анг слов! Удачи <b>{message.from_user.first_name}</b>'
bot.send_message(message.chat.id, mess, parse_mode='html')
#bot.message_handler(commands=['words'])
def website(message):
markup = types.ReplyKeyboardMarkup(resize_keyboard=True, row_width=1)
word = types.KeyboardButton('100 Слов')
markup.add(word)#создание самой кнопки.
bot.send_message(message.chat.id, 'Подевиться слова', reply_markup=markup)
#bot.message_handler(content_types=['text'])
def get_user_commands(message):
if message.text == '100 Слов':
url = 'https://www.kreekly.com/lists/100-samyh-populyarnyh-angliyskih-slov/'
response = requests.get(url)
soup = BeautifulSoup(response.text, 'lxml')
data = soup.find_all("div", class_="dict-word")
for i in data:
ENG = i.find("span", class_="eng").text
rU = i.find("span", class_="rus").text
bot.send_message(message.chat.id, ENG, parse_mode='html')
bot.send_message(message.chat.id, rU, parse_mode='html')
bot.polling(none_stop=True)
I tried to do it this way:
if i >= 20:
break
but this does not work. I also tried to do it like this:
if data.index(i) >= 20
break
Don't bother with the stopping the loop or slicing the array, tell bs4 to limit the result:
data = soup.find_all("div", class_="dict-word", limit=20)
you can try
for indx,i in enumerate(data):
if index == 19: # including zero, 20th will be the 19th.
break
-code here-
enumerate function keeps track of current step, so at first iteration indx = 0 , second indx=1 and so on.
for indx,i in enumerate(data):
if index < 20: # apply the code if it's under the 20th itereation.
-code here-
I suggest the upper solution.
When I run:
from bs4 import BeautifulSoup
import requests
import discord
from discord.ext import tasks
client = discord.Client()
#tasks.loop(minutes=1)
async def test():
channel = client.get_channel(973939538357522474)
await channel.send(takip)
#client.event
async def on_ready():
test.start()
async def takip():
url = ""
R = requests.get(url)
Soup = BeautifulSoup(R.text, "html5lib")
Title = Soup.find("h1", {"class": "pr-new-br"}).getText()
List = Soup.find("div", {"class": "pr-bx-nm with-org-prc"})
fiyat = List.find("span", {"class": "prc-dsc"}).getText()
degisenfiyat = float(fiyat.replace(",", ".").replace(" TL", ""))
if (degisenfiyat <= 200):
print("Fiyat düştü.")
client.run("")
I get:
A "<function takip at 0x00000244A7A440D0>" message in the discord channel
I want to use channel.send with the takip function. How do I do this?
takip is a function, takip() is what your function return
for example if you have this code
def my_sum(a, b):
return a + b
print(my_sum)
# This one print the function
# expected result : <function my_sum at 0x7f4dc82b7d30>
print(my_sum(1, 2))
# This one print what my function return (so 3 here)
# expected result : 3
In your code, you're sending your function to your discord channel, if you want to send "Fiyat düştü." if (degisenfiyat <= 200), you have to edit your code to this
from bs4 import BeautifulSoup
import requests
import discord
from discord.ext import tasks
client = discord.Client()
#tasks.loop(minutes=1)
async def test():
channel = client.get_channel(973939538357522474)
await channel.send(takip()) # Change here
#client.event
async def on_ready():
test.start()
def takip():
url = ""
R = requests.get(url)
Soup = BeautifulSoup(R.text, "html5lib")
Title = Soup.find("h1", {"class": "pr-new-br"}).getText()
List = Soup.find("div", {"class": "pr-bx-nm with-org-prc"})
fiyat = List.find("span", {"class": "prc-dsc"}).getText()
degisenfiyat = float(fiyat.replace(",", ".").replace(" TL", ""))
if (degisenfiyat <= 200):
return("Fiyat düştü.") # Change here
else:
return "degisenfiyat > 200"
client.run("")
However if (degisenfiyat > 200) this won't return anything so you will be sending None to your discord channel, I recommend you to add an else statement to return an error or more information (i.e.error : degisenfiyat > 200)
You're sending the function object. You need to call the function with parenthesis ()
This line:
await channel.send(takip)
Should be
await channel.send(takip())
takip() also needs to return a value to the caller, not print() it to the terminal. Use return instead of print:
if (degisenfiyat <= 200):
return "Fiyat düştü."
Consider this example of the function object, vs the returned value:
>>> def f():
... return "Hello"
...
>>> f()
'Hello'
>>> f
<function f at 0x103149fc0>
And finally, you need to remove the await from your function definition, as your call does not need to be asynchronous with the bot. This leaves your code at:
# ...
#tasks.loop(minutes=1)
async def test():
channel = client.get_channel(973939538357522474)
# Define x as the function's result before awaiting
x = takip()
await channel.send(x)
# ...
def takip():
url = ""
R = requests.get(url)
Soup = BeautifulSoup(R.text, "html5lib")
Title = Soup.find("h1", {"class": "pr-new-br"}).getText()
List = Soup.find("div", {"class": "pr-bx-nm with-org-prc"})
fiyat = List.find("span", {"class": "prc-dsc"}).getText()
degisenfiyat = float(fiyat.replace(",", ".").replace(" TL", ""))
if (degisenfiyat <= 200):
return "Fiyat düştü."
else:
return f"Error, degisenfiyat > 200. degisenfiyat = {degisenfiyat}"
client.run("")
async def on_message(message):
if message.content.startswith(prefix):
msg = message.content[20:]
else:
return None
if msg == "bitcoin" or "BITCOIN" or "btc" or "BTC":
url = "https://coinmarketcap.com/currencies/bitcoin/"
hdr = {'User-Agent': 'Mozilla/5.0'}
req = Request(url, headers=hdr)
res = urlopen(req).read()
soup = BeautifulSoup(res, 'html.parser')
btc_t1 = soup.find_all("div", class_="priceValue___11gHJ")
btc_t1 = [each_line.get_text().strip() for each_line in btc_t1[:20]]
btc_t1 = " ".join(btc_t1)
time.sleep(0.1)
btc_t2 = soup.find_all("span", class_="sc-1v2ivon-0 fiaaIx")
btc_t2 = [each_line.get_text().strip() for each_line in btc_t2[:20]]
btc_t2 = " ".join(btc_t2)
time.sleep(0.1)
btc_t3 = soup.find_all("div", class_="statsValue___2iaoZ")
btc_t3 = [each_line.get_text().strip() for each_line in btc_t3[:1]]
btc_t3 = " ".join(btc_t3)
time.sleep(0.1)
btcem = discord.Embed(title="Bitcoin", description="BTC market price \nPowered by Coinmarketcap", color=0xF7931A)
btcem.set_thumbnail(url="https://s2.coinmarketcap.com/static/img/coins/64x64/1.png")
btcem.add_field(name="Market Price", value="Price: "+ str(btc_t1) +" | "+ str(btc_t2), inline=False)
btcem.add_field(name="Market Cap", value="Price: "+ str(btc_t3), inline=False)
# embed.set_footer(text="Market", icon_url="")
await message.channel.send(embed=btcem)
if msg == "ethereum" or "ETHEREUM" or "eth" or "ETH":
url = "https://coinmarketcap.com/currencies/ethereum/"
hdr = {'User-Agent': 'Mozilla/5.0'}
req = Request(url, headers=hdr)
res = urlopen(req).read()
soup = BeautifulSoup(res, 'html.parser')
eth_t1 = soup.find_all("div", class_="priceValue___11gHJ")
eth_t1 = [each_line.get_text().strip() for each_line in eth_t1[:20]]
eth_t1 = " ".join(eth_t1)
time.sleep(0.1)
eth_t2 = soup.find_all("span", class_="sc-1v2ivon-0 fiaaIx")
eth_t2 = [each_line.get_text().strip() for each_line in eth_t2[:20]]
eth_t2 = " ".join(eth_t2)
time.sleep(0.1)
eth_t3 = soup.find_all("div", class_="statsValue___2iaoZ")
eth_t3 = [each_line.get_text().strip() for each_line in eth_t3[:1]]
eth_t3 = " ".join(eth_t3)
time.sleep(0.1)
ethem = discord.Embed(title="Ethereum", description="ETH market price \nPowered by Coinmarketcap", color=0x131313)
ethem.set_thumbnail(url="https://s2.coinmarketcap.com/static/img/coins/64x64/1027.png")
ethem.add_field(name="Market Price", value="Price: "+ str(eth_t1) +" | "+ str(eth_t2), inline=False)
ethem.add_field(name="Market Cap", value="Price: "+ str(eth_t3), inline=False)
# embed.set_footer(text="Market", icon_url="")
await message.channel.send(embed=ethem)
I'm trying to make Discord Coin, a stock bot in Python. All the modules used in the code have been installed, and I want to send crawl data by embed message, but when %bitcoin (prefix = %), the Ethereum embed along with the bitcoin embed also comes out.
Your if is completely messed up.
msg == "bitcoin" or "BITCOIN" or "btc" or "BTC" is always true.
Your check should be.
if msg in ('bitcoin', 'BITCOIN', 'btc', 'BTC')
and this also wouldn't work in your case,
since you are doing msg = msg[20:],
it should be msg = msg[1:].
Now, I directly debugged your code, this isn't the way to ask a question on SO. You should be able to debug your code and questions on SO should be based on your algorithm, technique or documentation.
see debugging
wouldn't it be easier to make a commands out of this?
like that(i havent tested it but it should work and it would be prettier):
bot = commands.Bot(command_prefix="%")
#bot.command(aliases=["BITCOIN", "btc","BTC"])
async def bitcoin(self, ctx):
# your bitcoin code here
#bot.command(aliases=["ETHEREUM", "eth", "ETH"])
async def ethereum(self, ctx):
# your etherum code here