I have this function which works fine locally on my machine with python 3.8, but it throws runtime error on Google Cloud Functions.
def telegram_test(request):
request_json = request.get_json()
import datetime
import pandas as pd
from pyrogram import Client
session_string = "...............38Q8uTHG5gHwyWD8nW6h................."
# the rest of the authantication
api_id = 32494131641215
api_hash = "ioadsfsjnjksfgnfriuthg#qw]/zwq ]w/\lc ec,"
# one of bbc channels on telegram you want to access
channel_name = 'pyrogram'
# if you only want to get messages older than 7 days in unix style
seven_days = int((datetime.datetime.now() - datetime.timedelta(days=7)).timestamp())
# call telegram with parameters such as limit and date
# save the result to dataframe
with Client(session_string,api_id,api_hash, takeout=True,workers=2) as app:
hist_iter = app.iter_history(channel_name,offset_date=seven_days, limit=100)
msglist = [msg.__dict__ for msg in hist_iter]
df = pd.DataFrame(msglist)
print(df.head(5))
return f'it works!:{request_json}'
The error message I get from GCF log:
File "/opt/python3.8/lib/python3.8/asyncio/events.py", line 639, in
get_event_loop raise RuntimeError('There is no current event loop in
thread %r.' RuntimeError: There is no current event loop in thread
'ThreadPoolExecutor-0_0'.
Update
I updated the code, the runtime error gone. but I am getting time out error.
I put the timeout 180 secondes, but still when I test the function times out on 60 seconds.
Here is the updated code. Is there something I am doing wrong?
async def foo():
from datetime import datetime, timedelta
from pandas import DataFrame
from pyrogram import Client
import asyncio
session_string = "********zNmkubA4ibjsdjhsdfjlhweruifnjkldfioY5DE*********"
api_id = 325511548224831351
api_hash = "jdffjgtrkjhfklmrtgjtrm;sesews;;wex"
channel_name = 'cnn'
with Client(session_string, api_id, api_hash, takeout=True) as app:
hist_iter = app.iter_history(
channel_name, limit=10)
msglist = [msg.__dict__ for msg in hist_iter]
df = DataFrame(msglist)
return df
async def bar():
return await foo()
def test(request):
from asyncio import run
return run(bar())
The solution in the end was to change from Pyrogram to telethon and create the asyncio manaually before creating the client.
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
Note: you need valid session string, otherwise when you test the function, it will wait for you to auth with mobile number. so first run this code locally and authenticate, then copy the session string to the cloud function.
Here is the full code:
from telethon.sessions import StringSession
from telethon import TelegramClient
from pandas import DataFrame
import datetime
import asyncio
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
api_id = 101010101
api_hash = "jhafcgahagfbahgdbw17171736456gerf"
session_string = "hjksdhjbdsfhgbdsabeyitrgdsbfsdbdiyfhsbddasbdjdksf="
channel_name = 'bbcuzbek'
seven_days = int((datetime.datetime.now() -
datetime.timedelta(days=7)).timestamp())
client = TelegramClient(StringSession(session_string),
api_id, api_hash, loop=loop)
time_format = "%d/%m/%Y, %H:%M:%S"
download_date = datetime.datetime.now(
tz=datetime.timezone.utc).strftime(time_format)
cols = ["id", "date", "text", "views", "download_date"]
async def foo():
all_msgs = [[message.id, message.date.strftime(time_format), message.text, message.views, download_date] async for message in client.iter_messages(entity=channel_name, offset_date=seven_days, limit=10)]
df = DataFrame(data=all_msgs, columns=cols)
# write it to BQ
# print(df)
# async for message in client.iter_messages(entity=channel_name, offset_date=seven_days, limit=10):
# print(message.id, message.date, message.text, message.views)
print("it runs")
print(len(df))
return None
def test(request):
with client:
return client.loop.run_until_complete(foo())
bar() is redundant
You're trying to return a dataframe. Is it a valid HTTP response?
with -> async with
hist_iter = app.iter_history() -> hist_iter = await app.iter_history()
M.b. it waits for input?
Related
I am using following code for scrapping messages from a telegram group but getting error;
Please guide me how to solve this or suggest any-other efficient solution.
RuntimeError: You must use "async with" if the event loop is running (i.e. you are inside an "async def")
Code:
from telethon.sync import TelegramClient
import datetime
import pandas as pd
import configparser
config = configparser.ConfigParser()
config.read("telethon.config")
api_id = config["telethon_credentials"]["api_id"]
api_hash = config["telethon_credentials"]["api_hash"]
chats = ['cryptodubai7']
client = TelegramClient('test', api_id, api_hash)
df = pd.DataFrame()
for chat in chats:
with TelegramClient('test', api_id, api_hash) as client:
for message in client.iter_messages(chat, offset_date=datetime.date.today() , reverse=True):
print(message)
data = { "group" : chat, "sender" : message.sender_id, "text" : message.text, "date" : message.date}
temp_df = pd.DataFrame(data, index=[1])
df = df.append(temp_df)
df['date'] = df['date'].dt.tz_localize(None)
You are creating 2 TelegramClient's, doesn't seem needed
You need to client.start()
Like the error suggest, you need to use async
Also an await is needed for retrieving the messages
I'd recommend using an event loop, like asyncio
Something like this should get you in the right direction:
Untested!
from telethon.sync import TelegramClient
import datetime
import pandas as pd
import asyncio
import configparser
config = configparser.ConfigParser()
config.read("telethon.config")
api_id = config["telethon_credentials"]["api_id"]
api_hash = config["telethon_credentials"]["api_hash"]
chats = ['cryptodubai7']
client = TelegramClient('test', api_id, api_hash)
client.start()
async def main():
df = pd.DataFrame()
for chat in chats:
messages = await client.iter_messages(chat, offset_date=datetime.date.today() , reverse=True)
for message in messages:
print(message)
data = { "group" : chat, "sender" : message.sender_id, "text" : message.text, "date" : message.date}
temp_df = pd.DataFrame(data, index=[1])
df = df.append(temp_df)
df['date'] = df['date'].dt.tz_localize(None)
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
I'm using peewee to store participants in a telegram channel. How do I get only new participants, i.e. those who have not been previously added?
Maybe we can offset by time? or offset by those who are already in the database?
Not so sure how to perform offsets in GetParticipantsRequest
from telethon import TelegramClient
from telethon.tl.functions.channels import GetParticipantsRequest
from telethon.tl.types import ChannelParticipantsSearch
from time import sleep
from schema import channel_users as cudb
from datetime import datetime
import json
from dotenv import load_dotenv
load_dotenv()
import os
api_id = os.getenv('API_ID')
api_hash = os.getenv('API_HASH')
PHONE = os.getenv('PHONE')
USERNAME = os.getenv('USERNAME')
# Remember to use your own values from my.telegram.org!
client = TelegramClient('anon', api_id, api_hash)
async def main():
# Getting information about yourself
me = await client.get_me()
my_channel = 'https://t.me/some_channel_url'
offset = 0
limit = 100
all_participants = []
while True:
participants = await client(GetParticipantsRequest(
my_channel, ChannelParticipantsSearch(''), offset, limit,
hash=0
))
if not participants.users:
break
all_participants.extend(participants.users)
offset += len(participants.users)
all_user_details = []
for participant in all_participants:
now = datetime.now()
date_added = now.strftime("%d/%m/%Y, %H:%M:%S")
channel_user_id, created = cudb.get_or_create(
id = participant.id,
defaults = {'first_name' : participant.first_name,
'last_name' : participant.last_name,
'username' : participant.username,
'phone' : participant.phone,
'is_bot' : participant.bot,
'date_added' : date_added}
)
if (created):
print(f'successfully created channel_usersID = {channel_user_id}')
else:
print(f'did not create anything, user information found in channel_usersID {channel_user_id}')
with client:
client.loop.run_until_complete(main())
ok I've sort of solved it with this. Problem is - Now trying to figure out how to update every time new user joins
while True:
participants = await client(GetParticipantsRequest(
my_channel, ChannelParticipantsSearch(''), offset, limit,
hash=0
))
number_of_participants = len(participants.users)
print(f'{len(participants.users)} length')
max_cudb = cudb.select(fn.MAX(cudb.channel_usersID)).scalar()
if max_cudb == len(participants.users):
print('id is same as number of participants in group, hence nothing new')
break
if not participants.users:
break
# calculate the difference between number of participants and last user added to DB
number_to_add = number_of_participants - max_cudb
# adds missing users chronologically from oldest to most recent
print(f'number_to_add = {number_to_add}')
for i in range(number_to_add-1,-1,-1):
print(f'i = {i}')
participant = participants.users[i]
now = datetime.now()
date_added = now.strftime("%d/%m/%Y, %H:%M:%S")
channel_user_id, created = cudb.get_or_create(
id = participant.id,
defaults = {'first_name' : participant.first_name,
'last_name' : participant.last_name,
'username' : participant.username,
'phone' : participant.phone,
'is_bot' : participant.bot,
'date_added' : date_added}
)
# Prints status of DB addition
if (created):
print(f'successfully created channel_usersID = {channel_user_id}')
else:
print(f'did not create anything, user information found in channel_usersID {channel_user_id}')
https://docs.telethon.dev/en/stable/quick-references/events-reference.html?highlight=chataction#chataction here you are the docs for chataction, exactly what you need just make sure to filter the event.
This is the first script which get's data from a website:
import requests
def get_prices():
name = "SeedifyFund"
crypto_data = requests.get("https://api.pancakeswap.info/api/tokens").json()["data"]
data = None
for i in crypto_data:
current = crypto_data[i]
if current['name'] == name:
data = {
"PriceUSD": current["price"],
"PriceBNB": current["price_BNB"],
}
return data
if __name__ == "__main__":
print(get_prices())
The code above outputs the following: {'PriceUSD': '1.022239219137518991087869433174527', 'PriceBNB': '0.002452203037583603303073246037795846'}
I'm having issue an issue with the second script. I want it to use the data that it has collected above and print it in a telegram bot when the user types /price. The code for the second script:
import telegram
from telegram.ext import Updater
from telegram.ext import CommandHandler
from tracker import get_prices
telegram_bot_token = "API TOKEN"
updater = Updater(token=telegram_bot_token, use_context=True)
dispatcher = updater.dispatcher
def price(update, context):
chat_id = update.effective_chat.id
message = ""
crypto_data = get_prices()
for i in crypto_data:
bnbprice = crypto_data[i]["pricebnb"]
usdprice = crypto_data[i]["priceusd"]
message += f"1 SFUND = \n${usdprice:,.2f} USD\n{bnbprice:.3f} BNB\n\n"
context.bot.send_message(chat_id=chat_id, text=message)
dispatcher.add_handler(CommandHandler("price", price))
updater.start_polling()
When the user types /price in the telegram chat it give this error:
coin = crypto_data[i]["pricebnb"]
TypeError: string indices must be integers
Could someone tell me what I'm doing wrong and help me solve the issue. Many thanks
I want to implement a parallel request.get() function, that processes a queue of requests and puts the result in a list, which, when finished, is processed by a standard sequential code. I tried the following, but my code doesn´t end and does not print the IDs.
import requests
from queue import Queue
from threading import Thread
BASE = 'http://www.uniprot.org'
KB_ENDPOINT = '/uniprot/'
FORMAT = ".xml"
num_threads = 10
ID_q = Queue()
ID_data = Queue()
# worker function
def get_ID_data(ID_q, ID_data, BASE, KB_ENDPOINT, FORMAT):
while True:
ID = ID_q.get()
print(ID)
ID_data.put(requests.get(BASE + KB_ENDPOINT + ID + FORMAT))
ID_q.task_done()
ID_data.task_done()
# initialize worker
for i in range(num_threads):
worker = Thread(target=get_ID_data, args=(ID_q, ID_data, BASE, KB_ENDPOINT, FORMAT))
worker.setDaemon(True)
worker.start()
# load IDs and put in queue
ID_list = ["A6ZMA9", "N1P5E6",
"H0GM11", "H0GZ91",
"A0A0L8VK54", "G2WKA0",
"C8ZEQ4", "B5VPH8",
"B3LLU5", "C7GL72",
"J8QFS9", "J8Q1C1",
"A0A0L8RDV1"]
for ID in ID_list:
ID_q.put(ID)
ID_q.join()
# work with ID_data
print(ID_data)
Update:
I changed #pkqxdd answer using asyncio and aiohttp to this:
import asyncio,aiohttp
IDs = ["A6ZMA9", "N1P5E6",
"H0GM11", "H0GZ91",
"A0A0L8VK54", "G2WKA0",
"C8ZEQ4", "B5VPH8",
"B3LLU5", "C7GL72",
"J8QFS9", "J8Q1C1",
"A0A0L8RDV1"]
BASE = 'http://www.uniprot.org'
KB_ENDPOINT = '/uniprot/'
FORMAT = ".xml"
async def get_data_coroutine(session, ID):
async with session.get(BASE + KB_ENDPOINT + ID + FORMAT) as response:
res = await response.text()
print(ID)
if not res:
raise NameError('{} is not available'.format(ID))
return res
async def main(loop):
async with aiohttp.ClientSession(loop=loop) as session:
tasks = [get_data_coroutine(session, ID) for ID in IDs]
return await asyncio.gather(*tasks)
loop = asyncio.get_event_loop()
result = loop.run_until_complete(main(loop))
Since you've mentioned async, I'm assuming you are using Python3.6 or higher.
The library requests doesn't really support async programming and it's kinda a dead end trying to make it async. A better idea is to use aiohttp instead.
You can achieve your goal with simple codes like this:
import asyncio,aiohttp
BASE = 'http://www.uniprot.org'
KB_ENDPOINT = '/uniprot/'
FORMAT = ".xml"
ID_list = ["A6ZMA9", "N1P5E6",
"H0GM11", "H0GZ91",
"A0A0L8VK54", "G2WKA0",
"C8ZEQ4", "B5VPH8",
"B3LLU5", "C7GL72",
"J8QFS9", "J8Q1C1",
"A0A0L8RDV1"]
session=aiohttp.ClientSession()
async def get_data(ID):
async with session.get(BASE + KB_ENDPOINT + ID + FORMAT) as response:
return await response.text()
coros=[]
for ID in ID_list:
coros.append(get_data(ID))
loop=asyncio.get_event_loop()
fut=asyncio.gather(*coros)
loop.run_until_complete(fut)
print(fut.result())
(Yes, I see the warning. But I don't really want to make the answer more complicated. You should change it to suit your purpose better.)
Help: Relatively new to telethon and was having issue with delete_messages()function. It "seems" to do nothing for me...
Initially I am getting the message id from send_message, from the return object's id value. It returns values like 1, 2, etc. Not sure this is the message id though.
I send it to delete_messages as delete_messages(channel, [id])
I get the channel (It's a private channel) from:
def resolve_channel_id(self, name):
try:
if name in self.__channel_ids:
return self.__channel_ids[name]
channel_id = self.client(ResolveUsernameRequest(name))
self.__channel_ids[name] = channel_id
return channel_id
# except UserNameNotOccupiedError as err:
except Exception as err:
# try for private channel
chatinvite = self.client(CheckChatInviteRequest(name))
channel_id = chatinvite.chat.id
access_hash_channel = chatinvite.chat.access_hash
channel_id = InputChannel(channel_id, access_hash_channel)
self.__channel_ids[name] = channel_id
return channel_id
I hope this example is useful for friends :
by this code, if a message with a Hello text is sent to a group with ID -1001300989487, that text will be deleted.
import logging
from telethon import TelegramClient, events
logging.basicConfig(level=logging.INFO)
api_id = XXXXXX
api_hash = 'XXXXXXXXXXXXXXXXXXXXXXX'
phone_number = '+989XXXXXXXX'
################################################
client = TelegramClient('session_name',
api_id,
api_hash,
)
#client.on(events.NewMessage(chats=(-1001300989487)))
async def my_event_handler(event):
if 'hello' in event.raw_text:
await event.delete()
client.start()
client.run_until_disconnected()
print('finish')
Works by updating to latest telethon: 1.0.3, and following documentation... https://telethon.readthedocs.io/en/latest/extra/basic/asyncio-magic.html