We Keep Coding

Python-telegram-bot bypass flood and 429 error using Scrapy

I follow the price drops on the target site. If there is a price decrease in accordance with the rules I have set, it is recorded in the notificate table. From there, a telegram notification is sent through the code I created in the pipelines.py file.
Sometimes the target site discounts too many products and 200 products can come from the notificate table. I'm stuck with telegram's flood protection while sending these.
Things I've tried:
1- Rotating multiple tokens
2- add sleep time
However, I cannot say that I was successful. I'm still stuck with the flood barrier.
What I want to do here is;
No matter how many notifications come from the Notificate table, queue them and send these notifications in a way that does not exceed 20 messages per second.
How can I do that.
My pipeline.py code:
def sendnotifications(self, token):
cursor = self.cnx.cursor()
req = requests
cursor.execute("SELECT * FROM notificate WHERE token= '"+token+"'")
notifications = cursor.fetchall()
for notification in notifications:
print(notification)
productid = notification[1]
url = notification[3]
name = notification[2]
old = notification[4]
new = notification[5]
price_difference = old - new
percentage = price_difference / old
percentage_str = str("%.2f" % (percentage * 100))
message = "<b>" + name + "</b>" + "\n\n" + \
str(old) + " TL >>>> " + \
str(new) + f" TL - {percentage_str}%" + "\n\n" + \
url + "\n\n" + \
if str(old) == "1.00" or str(old) == "2.00":
message = "<b>" + name + "</b>" + "\n\n" + \
"<b>" + str(new) + " TL\n\n" + "</b>" + \
url + "\n\n" + \
token_list = [
"xxxxxxxxxxxxxxxxxxxxxxx",
"yyyyyyyyyyyyyyyyyyyyyyyy",
"zzzzzzzzzzzzzzzzzzzzzzzzzzz",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
"ccccccccccccccccccccccccccccccccc",
]
TOKEN = token_list[random.randint(0, len(token_list)-1)]
chat_id = "-100xxxxxxxxxxxxx"
bot = telegram.Bot(token=TOKEN)
# tel_url = bot.sendMessage(chat_id = chat_id, text = message, parse_mode=ParseMode.HTML)
try:
bot.sendMessage(chat_id = chat_id, text = message, parse_mode=ParseMode.HTML)
sleep(0.05)
except Exception:
return False
cursor.close()
return True

The obvious way seems to be to chunk the updates into batches of 20 and sleep for more than 1 second between them:
# ...
notifications = cursor.fetchall()
cursor.close()
for i in range(0, len(notifications), 20):
chunk = notifications[i:i+20]
for notification in chunk:
print(notification)
# ...
time.sleep(1.5)

Table values not getting inserted into mysql database until cntrl+c is entered to close python file in linux

Edited
Q)Can someone help getting the values inserted into mysql database , just confused where place mydb function
Reason :Once I manually enter cntrl+c for .py , then only the values are getting inserted into mysql database
Used in the .py file
here is the complete code , where should i place the mydb function?
Table values not getting inserted into mysql database until cntrl+c is entered to close python file in linux
import os
import re
from builtins import len, Exception
import slack
import logging
from subprocess import check_output
import datetime
import mysql.connector
import time
import json
import requests
#user_threads_info = {}
#thread_ts = ""
#slack.RTMClient.run_on(event='message')
def say_hello(**payload):
try:
##0 get clients and payload
logging.info('msg received')
data = payload['data']
web_client = payload['web_client']
rtm_client = payload['rtm_client']
##0 - 1 Check if it is the first msg, not replied msg by me
# print(data)
if data.get('text') == None:
logging.info('This msg is my replied msg.')
return False
##0-2 Get channel info
channel_id = data['channel']
thread_ts = data['ts']
global user
user = data['user']
#user_info = get_userinfo(user)
#print(user_info)
msg = data['text']
##1 get scenario submsg
retVal = analysis_msg(msg)
# print(retVal)
response = web_client.users_list()
assert(response['ok'])
user_map = {x['id']: x['name'] for x in response['members']}
global user_name
user_name = user_map[user] if user in user_map else None
print(user_name)
if retVal[0] == False:
retMsg = retVal[1] + "\nI can create the following orders. \n" \
"a) spu - store pickup \n" \
"b) sth - ship to home \n" \
"c) del - delivery \n" \
"d) digitalAsGuest - Digital item \n" \
" \n" \
"Please provide information as mentioned in below example.\n" \
" \n" \
"Example: spu:3646989:sftqa3:AMEX\n" \
"\n" \
"Sample SKUS:\n" \
"spu - [3646989,8862011]\n" \
"sth - [2592015,6140094]\n" \
"del - [5592005,8862011]\n" \
"digitalAsGuest - [2810037,5057400]"
send_msg(web_client, channel_id, thread_ts, user, retMsg)
return False
##2 form cmd
retVal = form_cmd(retVal[1])
print(retVal)
if retVal == False:
return False
##3 execute cmd
# inform the start of test
retMsg = "Creating an order,Please wait for the result."
send_msg(web_client, channel_id, thread_ts, user, retMsg)
global res
try:
res1 = os.popen(retVal).read()
print("Printing result...")
print(res1)
print("end of print")
res = reg_result_new(res1)
if res == False:
print("reg_function failure")
retMsg = "The test order placement failed."
else:
retMsg = "Order Id - " + res['id'] + "\nFirst Name - " + res['firstName'] + "\nLast Name - " + res['lastName'] + "\n PhoneNumber - " + res['dayPhoneNumber'] + "\n Email - " + res['email'] + "\n"
except Exception as ee:
retMsg = "The test scenario has a failure. Please Check the feature file."
## 4 send result to slack
# retMsg = "Order Id - " + res['id'] + "\nFirst Name - " + res['firstName'] + "\nLast Name - " + res['lastName'] + "\n PhoneNumber - " + res['day PhoneNumber'] + "\n Email - " + res['email'] + "\n"
create_result_file(user, res)
send_msg(web_client, channel_id, thread_ts, user, retMsg)
print(retVal)
except Exception as e:
print("error")
logging.critical(str(e))
############################ My handlers ##############################
def create_result_file(user, res):
try:
cur_time = datetime.datetime.now()
file_name = user + str(cur_time.year) + str(cur_time.month) + str(cur_time.day) + str(cur_time.hour) + str(
cur_time.minute) + str(cur_time.second) + '.txt'
file = open(file_name, 'w')
file.write(res)
file.close()
except Exception as e:
print(str(e))
def send_msg(web_client, channel_id, thread_ts,user,mgs):
print("thread_ts value is:"+thread_ts)
web_client.chat_postMessage(
channel=channel_id,
text=f"```Hi <#{user}>! \n " + mgs + "```",
thread_ts=thread_ts
)
#def get_userinfo(user):
# payload = {'token': slack_token, 'user': user}
# r = requests.get('https://slack.com/api/users.info', params=payload)
# print(r.text)
# return json.loads(r.text)["user"]
# error code mgmt.
def error_code(code):
# reserved
print(code)
return [False, code]
# break down msg to the test scenario submsgs
def analysis_msg(msg):
global submsg
submsg = msg.split(":")
for value in submsg:
print(value)
if len(submsg) != 4:
logging.warning("This msg not test scenario")
return error_code("Please check the format")
res = {}
res["feature"] = submsg[0]
res["sku"] = submsg[1]
res["env"] = submsg[2]
res["payment"] = submsg[3]
###check
if validate_sku(res["sku"]) == False:
return error_code("INVALID_SKU \n")
if validate_env(res["env"]) == False:
return error_code("INVALID_ENV \n")
if validate_payment(res["payment"]) == False:
return error_code("INVALID_payment \n")
if check_specialCharacter(res["feature"]) == False:
return error_code("INVALID_PROFILE_WITH_SPECIAL_CHARACTER")
return [True, res]
# form cmd for test bat files ! reserved
def form_cmd(submsg):
cmd = 'sh /home/iptbot/iptautobot/test.sh ' + submsg['env'] + ' ' + submsg['feature'] + ' ' + submsg["sku"] + ' ' + submsg["payment"]
return cmd
#code to print user details
#code to print user details
def reg_result_new(res):
start = 'COP Order Response :'
end = 'isGuestMode'
start_index = res.find(start) + len(start)
res = res[start_index:]
end_index = res.find(end) + 22
global data
data = res[:end_index]
try:
print('Data -> ' + str(data))
data = json.loads(data.strip())
new_data = {}
new_data['id'] = data['id']
new_data['firstName'] = data['lineItems'][0]['fulfillmentInfo']['storeInfo']['agentInfo']['firstName']
new_data['lastName'] = data['lineItems'][0]['fulfillmentInfo']['storeInfo']['agentInfo']['lastName']
new_data['dayPhoneNumber'] = data['lineItems'][0]['fulfillmentInfo']['storeInfo']['agentInfo']['dayPhoneNumber']
new_data['email'] = data['lineItems'][0]['fulfillmentInfo']['storeInfo']['agentInfo']['email']
#new_data['firstName'] = data['paymentInfo']['billingAddressInfo']['firstName']
return new_data
except Exception as e:
print('Here error -> '+str(e))
return False
#def reg_result(res):
# "COP Order Response"
# lines = res.split('\n')
# for line in lines:
# pattern = "COP Order Response*"
# prog = re.compile(pattern)
# result = prog.search(line)
# if result == None:
# continue
# res1 = result.string.split('{')
# if len(res1) < 2:
# continue
# res2 = res1[1].split(',')
# if len(res2) < 2:
# continue
# res3 = res2[0].split(':')
# if len(res3) < 2:
# continue
# return res3[1]
# COP Order Response : {"id":"BBY01-200001878853"
# return False
# return val is Boolean
# True/False
# Input type: String
# for positive integer only
# alternative way: Handle exception for int(d)
def validate_sku(sku_val):
return sku_val.isnumeric()
# input val : string
# return val: Boolean
def validate_env(env_val):
env_list = [
"sftqa1" , "sftqa2" , "sftqa3" , "sftqa4"
]
if env_val in env_list:
return True
else:
return False
def validate_payment(payment_val):
env_payment = [
"AMEX","VISA"
]
if payment_val in env_payment:
return True
else:
return False
# input val : string
# return val: Boolean
def check_specialCharacter(s):
if s == "":
return False
if s.isspace():
return False
return s.isalnum()
slack_token = os.environ["SLACK_API_TOKEN"]
rtm_client = slack.RTMClient(token=slack_token)
rtm_client.start()
#database connction
mydb = mysql.connector.connect(
host="host",
user="user",
passwd="pass",
database="db"
)
mycursor = mydb.cursor()
for value in submsg:
print(value)
fulfilment=submsg[0]
sku=submsg[1]
environment=submsg[2]
payment=submsg[3]
ts = time.time()
date = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
orderNumber=data['id']
username=user_name
print(fulfilment)
print(sku)
print(environment)
print(payment)
print(username)
print(orderNumber)
sqlformula = "INSERT INTO orderDetails (fulfilment,sku,environment,payment,orderNumber,date,user) VALUES (%s,%s,%s,%s,%s,%s,%s)"
#order=("sth",3643387,"sftqa2","AMEX")
#mycursor.execute(sqlformula,order)
mycursor.execute(sqlformula,(fulfilment,sku,environment,payment,orderNumber,date,username))
mydb.commit()
mydb.close()
Output
1 sh /home/iptbot/iptautobot/test.sh sftqa3 spu 3646989 AMEX
2 error
3 CRITICAL:root:'user'
4 error
5 CRITICAL:root:'user' // clicking Control+C values get inserted
6 ^CWARNING:slack.rtm.client:Websocket was closed.
7 3646989
8 sftqa3
9 AMEX
10 spu
11 3646989
12 sftqa3
13 AMEX
14 a6002043
15 BBY01-200002091354

You are stuck at this point because rtm_client.start() is a synchronous call.
If you want it to be asynchronous (non-blocking) then you should run:
rtm_client.start(run_async=True)
Here it is good walk-through on how to setup async usage of the library. Also have a look at the method signature for RTMClient to get an idea of how it works.
Here's a good example detailing a lot of what you would need in your case.
Then you will hit your db execution code where you will need to have a while loop to go through the data you want to add to the DB.
I would recommend that you use a Queue for this as it is synchronised and will be easier to manage than a global list which is overwritten on every order. Preferably you could use asyncio.Queue with an example of implementation here
When an order has passed the validation steps add it to the queue. Here is some pseudo code describing the flow with a basic (not asyncio) Queue:
import queue
q = queue.Queue()
def validate_order(order):
valid_order_data = ......
q.put(valid_order_data)
while True:
valid_order = q.get() # Will wait until there is a value on the queue
mycursor.execute(sqlformula, (valid_order))

os not letting me save email attachment using pywin32

import win32com.client
import pythoncom
import time
import os
class Handler_Class(object):
def OnNewMailEx(self, receivedItemsIDs):
# RecrivedItemIDs is a collection of mail IDs separated by a ",".
# You know, sometimes more than 1 mail is received at the same moment.
for ID in receivedItemsIDs.split(","):
print('Running scan...')
mail = outlook.Session.GetItemFromID(ID)
email_date = mail.SentOn.strftime("%m-%d-%y" + " at " + "%I:%M:%S %p")
email_date_stamp = mail.SentOn.strftime('%m-%d-%y_%I:%M:%S-%p')
email_message = mail.Body
email_subject = mail.Subject
email_sender = mail.SenderEmailAddress
email_attachments = mail.Attachments
try:
if check_correct_subject(email_subject) == True:
print('From: ' + email_sender)
print('Subject: ' + email_subject)
print('Date: ' + email_date)
if email_attachments.Count > 0:
print(str(email_attachments.Count) + ' attachments found.')
for i in range(email_attachments.Count):
email_attachment = email_attachments.Item(i + 1)
report_name = email_date_stamp + '_' + email_attachment.FileName
print(report_name)
email_attachment.SaveASFile(os.getcwd() + '\\Reports\\Broker_Risk_LDW\\' + report_name)
print('Pushing attachment - ' + report_name + ' - to check_correct_email() function.')
if check_correct_attachment(email_attachment) == True:
save_incoming_report(email_attachment, report_name, get_report_directory(email_subject))
else:
print('Not the attachment we are looking for.')
# add error logging here
break
else: # ***********add error logging here**************
print('No attachment found.')
except:
pass
I am calling the class like this.
if __name__ == '__main__':
outlook = win32com.client.DispatchWithEvents("Outlook.Application", Handler_Class)
#and then an infinit loop that waits from events.
pythoncom.PumpMessages()
The problem here is this line: email_attachment.SaveASFile(os.getcwd() + '\\Reports\\Broker_Risk_LDW\\' + report_name)
That folder is already created and it was working before I switched this to a class, in order to work with an Outlook event listener. I am not getting an error message, however the attachment is not saving.
Any tips?

Identify sending user, Python IRC

So I made a Twitch.tv bot for my own channel, after having fun with it a little bit, I wanted to have some command restricted to some users, and some commands that can say the users name, for example:
Username reply example:
Person1: !tea
PythonBot: Would you like some tea, Person1?
Admin restriction example:
Person1: !ban Person2
PythonBot: I'm sorry, Person1, This command is restricted to admins only.
Ok, So here is the code I'm using (I will be modifying it soon to make it my own)
import socket
import threading
bot_owner = '~Not Today~'
nick = '~Not Today~'
channel = '~Not Today~'
server = 'irc.twitch.tv'
password = '~Not Today~'
queue = 13
irc = socket.socket()
irc.connect((server, 6667))
irc.send('PASS ' + password + '\r\n')
irc.send('USER ' + nick + ' 0 * :' + bot_owner + '\r\n')
irc.send('NICK ' + nick + '\r\n')
irc.send('JOIN ' + channel + '\r\n')
def message(msg):
global queue
queue = 5
if queue < 20:
irc.send('PRIVMSG' + channel + ' :' + msg + '\r\n')
else:
print 'Message Deleted'
def queuetimer():
global queue
queue = 0
threading.Timer(30,queuetimer).start()
queuetimer()
while True:
botdata = irc.recv(1204)
botuser = botdata.split(':')[1]
botuser = botuser.split('!')[0]
print botdata
if botdata.find('PING') != -1:
irc.send(botdata.replace('PING', 'PONG'))
if botdata.find('!res') != -1:
irc.send(botdata.replace('!res', '1600x900'))

The twitch IRC raw message is like
:jkm!jkm#jkm.tmi.twitch.tv PRIVMSG #trumpsc :needs Kappa
for above msg, it actually means userjkm at channel trumpsc saying needs Kappa
for your code, the method to get botuser is right, but you don't have the message the user sent, add following code should get the message
botmsg = botdata.split(':')[2]
so you get the message and username, the next step would be handling them.
Here would be some example for your need. For me, I will prepared a adminuserList and commmandList for other command, but I will simplify it here.
def commmanHandler(botuser, botmsg): # botmsg = '!ban user1'
command = botmsg.split(' ')[0] # command = '!ban'
command = command[1:] # command = 'ban'
argu = message.split(' ')[1] # argu = 'user1'
if command not in commmandList:
raise Exception("command not support")
if command == 'ban': # ban command, or using switch
# check if admin
if botuser not in adminList:
raise Exception("I'm sorry, " + botuser + ", This command is restricted to admins only.")
# admin, able to ban
irc.send('PRIVMSG' + channel + ' :' + '.ban ' + argu)
then call this function in your while loop to handle all message you get
try:
commmanHandler(botuser, botmsg)
except Exception, e:
print e
irc.send('PRIVMSG' + channel + ' :' + e)
here would be my solution, and also, don't forget to give the bot moderator privilege.

Python smtplib sometimes fails sending

I wrote a simple "POP3S to Secure SMTP over TLS" MRA script in Python (see below).
It works fine, but sometimes it returns "Connection unexpectedly closed" while trying to send via SMTP. Running the script again will deliver that message successfully.
Please give me some suggestions why it would fail to deliver a message sometimes but at the next run it delivers exactly this message successfully!?
#! /usr/bin/env python
import poplib
import email
def forward_pop3_smtp( smtp_credentials, pop3_credentials, forward_address):
pop3_server = pop3_credentials[0]
pop3_port = pop3_credentials[1]
pop3_user = pop3_credentials[2]
pop3_password = pop3_credentials[3]
message_recipient = forward_address
server = poplib.POP3_SSL( pop3_server, pop3_port)
server.user( pop3_user)
server.pass_( pop3_password)
for messages_iterator in range( len( server.list()[1])):
message_list = server.retr( messages_iterator + 1)[1]
message_string = ''
for message_line in message_list:
message_string += message_line + '\n'
message_message = email.message_from_string( message_string)
message_message_as_string = message_message.as_string()
message_sender = message_message[ 'From']
print( 'message_sender = ' + message_sender)
smtp_return = send_smtp( smtp_credentials, message_sender, message_recipient, message_message_as_string)
print( 'smtp_return = ' + str(smtp_return))
if smtp_return == 0:
print( 'Deleting message ' + message_message[ 'Subject'] + ':\n')
return_delete = server.dele( messages_iterator + 1)
print( 'return_delete = \n' + str(return_delete))
print( '\n')
server.quit()
def send_smtp( smtp_credentials, message_sender, message_recipient, message_message_as_string):
smtp_server = smtp_credentials[0]
smtp_port = smtp_credentials[1]
smtp_user = smtp_credentials[2]
smtp_password = smtp_credentials[3]
import smtplib
exception = 0
try:
server = smtplib.SMTP( smtp_server)
server.starttls()
server.login( smtp_user, smtp_password)
smtp_sendmail_return = server.sendmail( message_sender, message_recipient, message_message_as_string)
server.quit()
except Exception, e:
exception = 'SMTP Exception:\n' + str( e) + '\n' + str( smtp_sendmail_return)
return exception
if __name__ == '__main_':
print( 'This module needs to be imported!\n')
quit()

Use Port 587 for TLS. I don't see the script use smtp_port
Use like,
server = smtplib.SMTP( smtp_server, int(smtp_port)
For Secure SMTP (SMTP + SSL), use smtplib.SMTP_SSL