We Keep Coding

Table values not getting inserted into mysql database until cntrl+c is entered to close python file in linux

Edited
Q)Can someone help getting the values inserted into mysql database , just confused where place mydb function
Reason :Once I manually enter cntrl+c for .py , then only the values are getting inserted into mysql database
Used in the .py file
here is the complete code , where should i place the mydb function?
Table values not getting inserted into mysql database until cntrl+c is entered to close python file in linux
import os
import re
from builtins import len, Exception
import slack
import logging
from subprocess import check_output
import datetime
import mysql.connector
import time
import json
import requests
#user_threads_info = {}
#thread_ts = ""
#slack.RTMClient.run_on(event='message')
def say_hello(**payload):
try:
##0 get clients and payload
logging.info('msg received')
data = payload['data']
web_client = payload['web_client']
rtm_client = payload['rtm_client']
##0 - 1 Check if it is the first msg, not replied msg by me
# print(data)
if data.get('text') == None:
logging.info('This msg is my replied msg.')
return False
##0-2 Get channel info
channel_id = data['channel']
thread_ts = data['ts']
global user
user = data['user']
#user_info = get_userinfo(user)
#print(user_info)
msg = data['text']
##1 get scenario submsg
retVal = analysis_msg(msg)
# print(retVal)
response = web_client.users_list()
assert(response['ok'])
user_map = {x['id']: x['name'] for x in response['members']}
global user_name
user_name = user_map[user] if user in user_map else None
print(user_name)
if retVal[0] == False:
retMsg = retVal[1] + "\nI can create the following orders. \n" \
"a) spu - store pickup \n" \
"b) sth - ship to home \n" \
"c) del - delivery \n" \
"d) digitalAsGuest - Digital item \n" \
" \n" \
"Please provide information as mentioned in below example.\n" \
" \n" \
"Example: spu:3646989:sftqa3:AMEX\n" \
"\n" \
"Sample SKUS:\n" \
"spu - [3646989,8862011]\n" \
"sth - [2592015,6140094]\n" \
"del - [5592005,8862011]\n" \
"digitalAsGuest - [2810037,5057400]"
send_msg(web_client, channel_id, thread_ts, user, retMsg)
return False
##2 form cmd
retVal = form_cmd(retVal[1])
print(retVal)
if retVal == False:
return False
##3 execute cmd
# inform the start of test
retMsg = "Creating an order,Please wait for the result."
send_msg(web_client, channel_id, thread_ts, user, retMsg)
global res
try:
res1 = os.popen(retVal).read()
print("Printing result...")
print(res1)
print("end of print")
res = reg_result_new(res1)
if res == False:
print("reg_function failure")
retMsg = "The test order placement failed."
else:
retMsg = "Order Id - " + res['id'] + "\nFirst Name - " + res['firstName'] + "\nLast Name - " + res['lastName'] + "\n PhoneNumber - " + res['dayPhoneNumber'] + "\n Email - " + res['email'] + "\n"
except Exception as ee:
retMsg = "The test scenario has a failure. Please Check the feature file."
## 4 send result to slack
# retMsg = "Order Id - " + res['id'] + "\nFirst Name - " + res['firstName'] + "\nLast Name - " + res['lastName'] + "\n PhoneNumber - " + res['day PhoneNumber'] + "\n Email - " + res['email'] + "\n"
create_result_file(user, res)
send_msg(web_client, channel_id, thread_ts, user, retMsg)
print(retVal)
except Exception as e:
print("error")
logging.critical(str(e))
############################ My handlers ##############################
def create_result_file(user, res):
try:
cur_time = datetime.datetime.now()
file_name = user + str(cur_time.year) + str(cur_time.month) + str(cur_time.day) + str(cur_time.hour) + str(
cur_time.minute) + str(cur_time.second) + '.txt'
file = open(file_name, 'w')
file.write(res)
file.close()
except Exception as e:
print(str(e))
def send_msg(web_client, channel_id, thread_ts,user,mgs):
print("thread_ts value is:"+thread_ts)
web_client.chat_postMessage(
channel=channel_id,
text=f"```Hi <#{user}>! \n " + mgs + "```",
thread_ts=thread_ts
)
#def get_userinfo(user):
# payload = {'token': slack_token, 'user': user}
# r = requests.get('https://slack.com/api/users.info', params=payload)
# print(r.text)
# return json.loads(r.text)["user"]
# error code mgmt.
def error_code(code):
# reserved
print(code)
return [False, code]
# break down msg to the test scenario submsgs
def analysis_msg(msg):
global submsg
submsg = msg.split(":")
for value in submsg:
print(value)
if len(submsg) != 4:
logging.warning("This msg not test scenario")
return error_code("Please check the format")
res = {}
res["feature"] = submsg[0]
res["sku"] = submsg[1]
res["env"] = submsg[2]
res["payment"] = submsg[3]
###check
if validate_sku(res["sku"]) == False:
return error_code("INVALID_SKU \n")
if validate_env(res["env"]) == False:
return error_code("INVALID_ENV \n")
if validate_payment(res["payment"]) == False:
return error_code("INVALID_payment \n")
if check_specialCharacter(res["feature"]) == False:
return error_code("INVALID_PROFILE_WITH_SPECIAL_CHARACTER")
return [True, res]
# form cmd for test bat files ! reserved
def form_cmd(submsg):
cmd = 'sh /home/iptbot/iptautobot/test.sh ' + submsg['env'] + ' ' + submsg['feature'] + ' ' + submsg["sku"] + ' ' + submsg["payment"]
return cmd
#code to print user details
#code to print user details
def reg_result_new(res):
start = 'COP Order Response :'
end = 'isGuestMode'
start_index = res.find(start) + len(start)
res = res[start_index:]
end_index = res.find(end) + 22
global data
data = res[:end_index]
try:
print('Data -> ' + str(data))
data = json.loads(data.strip())
new_data = {}
new_data['id'] = data['id']
new_data['firstName'] = data['lineItems'][0]['fulfillmentInfo']['storeInfo']['agentInfo']['firstName']
new_data['lastName'] = data['lineItems'][0]['fulfillmentInfo']['storeInfo']['agentInfo']['lastName']
new_data['dayPhoneNumber'] = data['lineItems'][0]['fulfillmentInfo']['storeInfo']['agentInfo']['dayPhoneNumber']
new_data['email'] = data['lineItems'][0]['fulfillmentInfo']['storeInfo']['agentInfo']['email']
#new_data['firstName'] = data['paymentInfo']['billingAddressInfo']['firstName']
return new_data
except Exception as e:
print('Here error -> '+str(e))
return False
#def reg_result(res):
# "COP Order Response"
# lines = res.split('\n')
# for line in lines:
# pattern = "COP Order Response*"
# prog = re.compile(pattern)
# result = prog.search(line)
# if result == None:
# continue
# res1 = result.string.split('{')
# if len(res1) < 2:
# continue
# res2 = res1[1].split(',')
# if len(res2) < 2:
# continue
# res3 = res2[0].split(':')
# if len(res3) < 2:
# continue
# return res3[1]
# COP Order Response : {"id":"BBY01-200001878853"
# return False
# return val is Boolean
# True/False
# Input type: String
# for positive integer only
# alternative way: Handle exception for int(d)
def validate_sku(sku_val):
return sku_val.isnumeric()
# input val : string
# return val: Boolean
def validate_env(env_val):
env_list = [
"sftqa1" , "sftqa2" , "sftqa3" , "sftqa4"
]
if env_val in env_list:
return True
else:
return False
def validate_payment(payment_val):
env_payment = [
"AMEX","VISA"
]
if payment_val in env_payment:
return True
else:
return False
# input val : string
# return val: Boolean
def check_specialCharacter(s):
if s == "":
return False
if s.isspace():
return False
return s.isalnum()
slack_token = os.environ["SLACK_API_TOKEN"]
rtm_client = slack.RTMClient(token=slack_token)
rtm_client.start()
#database connction
mydb = mysql.connector.connect(
host="host",
user="user",
passwd="pass",
database="db"
)
mycursor = mydb.cursor()
for value in submsg:
print(value)
fulfilment=submsg[0]
sku=submsg[1]
environment=submsg[2]
payment=submsg[3]
ts = time.time()
date = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
orderNumber=data['id']
username=user_name
print(fulfilment)
print(sku)
print(environment)
print(payment)
print(username)
print(orderNumber)
sqlformula = "INSERT INTO orderDetails (fulfilment,sku,environment,payment,orderNumber,date,user) VALUES (%s,%s,%s,%s,%s,%s,%s)"
#order=("sth",3643387,"sftqa2","AMEX")
#mycursor.execute(sqlformula,order)
mycursor.execute(sqlformula,(fulfilment,sku,environment,payment,orderNumber,date,username))
mydb.commit()
mydb.close()
Output
1 sh /home/iptbot/iptautobot/test.sh sftqa3 spu 3646989 AMEX
2 error
3 CRITICAL:root:'user'
4 error
5 CRITICAL:root:'user' // clicking Control+C values get inserted
6 ^CWARNING:slack.rtm.client:Websocket was closed.
7 3646989
8 sftqa3
9 AMEX
10 spu
11 3646989
12 sftqa3
13 AMEX
14 a6002043
15 BBY01-200002091354

You are stuck at this point because rtm_client.start() is a synchronous call.
If you want it to be asynchronous (non-blocking) then you should run:
rtm_client.start(run_async=True)
Here it is good walk-through on how to setup async usage of the library. Also have a look at the method signature for RTMClient to get an idea of how it works.
Here's a good example detailing a lot of what you would need in your case.
Then you will hit your db execution code where you will need to have a while loop to go through the data you want to add to the DB.
I would recommend that you use a Queue for this as it is synchronised and will be easier to manage than a global list which is overwritten on every order. Preferably you could use asyncio.Queue with an example of implementation here
When an order has passed the validation steps add it to the queue. Here is some pseudo code describing the flow with a basic (not asyncio) Queue:
import queue
q = queue.Queue()
def validate_order(order):
valid_order_data = ......
q.put(valid_order_data)
while True:
valid_order = q.get() # Will wait until there is a value on the queue
mycursor.execute(sqlformula, (valid_order))

Python Include contents of If/else in output not just print

I am not that experienced with python, but do some python coding for small jobs. Currently I have a job that opens a log file and pulls any records that are considered errors. This list of errors is then added as part of an email notification. What I would like to do is either include the list or a notification that the list was empty. I have been able to do this in the console, but don't know how to get this added as a parameter in the email.
if errorlist:
print "\n".join(errorlist)
else:
print "No Errors Found"
# Send Email
SMTP_SERVER = {SMTP SERVER}
SMTP_PORT = {SMTP PORT}
sender = {Sender}
password = {Password}
recipient = {Recipient}
subject = "This is the subject line"
errorlist = "<br>" "\n".join(errorlist)
body = "" + errorlist + ""
headers = ["From: " + sender,
"Subject: " + subject,
"To: " + ", " .join(recipient),
"MIME-Version: 1.0",
"Content-Type: text/html"]
headers = "\r\n".join(headers)
session = smtplib.SMTP(SMTP_SERVER, SMTP_PORT)
session.ehlo()
session.starttls()
session.ehlo
session.login(sender, password)
session.sendmail(sender, recipient, headers + "\r\n\r\n" + body)
session.quit()

The email is sent in this line:
session.sendmail(sender, recipient, headers + "\r\n\r\n" + body)
The body variable contains the body of your email. In order to add something into the body of the email, it should be added to the string contained by the body variable. Adapting the code you already have added (which successfully prints your desired result), you could replace this line:
body = "" + errorlist + ""
with this:
if errorlist:
body = "\n".join(errorlist)
else:
body = "No Errors Found"

if errorlist:
error_string = "\n".join(errorlist) # assign it to variable
print (error_string) # still print it
else:
error_string = "" # assign blank to error_string
print ("No Errors Found") # still print "no errors found"
.
.
.
body = ""+error_string+"" # 'body = error_string' is the same though
.
.
.
session.sendmail(sender, recipient, headers + "\r\n\r\n" + body) # this line you could replace "body" with "error_string" because they are pretty much goign to be equivilant because of the previous comment
You want to assign your error string into a variable, and then use the variable later when constructing the body. Also there is more room for simplification

Type error: 'module' object has no attribute '_getitem_'

I am trying to fetch emails and then we need to write the mails in to different files according to the subject.
import email
import imaplib
from threading import Thread
from time import sleep
import time
def myThreadFun():
M = imaplib.IMAP4_SSL('imap.gmail.com')
M.login("noticeboard16#gmail.com", "embeddedSystems")
while (1):
M.select()
rv, data1 = M.search(None, 'UNSEEN')
for i in data1[0].split():
resp, data = M.FETCH(i, '(RFC822)')
mail = email.message_from_string(data[0][1])
for part in mail.walk():
# multipart are just containers, so we skip them
if part.get_content_maintype() == 'multipart':
continue
# we are interested only in the simple text messages
if part.get_content_subtype() != 'plain':
continue
payload = part.get_payload()
print '\n'
print '[%s]' % (mail['Subject'])
print 'From: %s' % (mail['From'])
print 'Date:', mail['Date']
print '================================='
print payload
#time.sleep(10)
#save_string = str("/home/buddhi-xubuntu/Python/email_" + ".text")
#myfile = open(save_string, 'a')
#myfile.write(mail['Subject']+ "\nFrom: " + mail['From'] + "\nDate: " + mail['Date'] + "=============\n" + payload)
#myfile.close()
#time.sleep(10)
#with file('email_.txt', 'r') as original: data = original.read()
#with file('email_2.txt', 'w') as modified: modified.write(mail['Subject']+ "\nFrom: " + mail['From'] + "\nDate: " + mail['Date'] + "\n=============\n" + payload + "\n" + data)
#orig_string = str("/home/e11452/Downloads/email_" + ".text")
#f = open(orig_string,'r')
#temp = f.read()
#f.close()
if mail['Subject']=="E/11":
new_string = str("/home/e11452/Downloads/email_11" + ".text")
f = open(new_string, 'w')
f.write(mail['Subject']+ "\nFrom: " + mail['From'] + "\nDate: " + mail['Date'] + "\n=============\n" + payload + "\n")
elif mail['Subject']=="E/10":
new_string = str("/home/e11452/Downloads/email_12" + ".text")
-f = open(new_string, 'w')
f.write(mail['Subject']+ "\nFrom: " + mail['From'] + "\nDate: " + mail['Date'] + "\n=============\n" + payload + "\n")
f.write(temp)
f.close()
time.sleep(10)
M.LOGOUT()
thread = Thread(target = myThreadFun)
thread.start()
above is the code i tried and i get an error saying
Traceback (most recent call last): File "email14.py", line 58, in if email['Subject'] == 'E/11': TypeError: 'module' object has no attribute 'getitem'

Seems that you misspell mail as email. email is a module you imported. Then you got the error.

Using python and elixir to send emails utilizing db tables with foreign keys

I have a python script that pulls information from a single database table and sends a weekly update email to users. I'd like to find a way to modify my code to allow for normalizing the structure and putting all of the investigator info (prefix, fname, lname, and email) into it's own table. However, I'm not sure how to do this within the structure of the elixir model and my python code.
Here is my current elixir model file:
from elixir import *
import auth as auth
au = auth.UserAuth()
metadata.bind = 'mysql://' + au.user + ':' + au.password + '#localhost/' + au.database
metadata.bind.echo = True
class Protocol(Entity):
id = Field(Integer, primary_key = True)
irb_no = Field(Text)
title = Field(Text)
prefix = Field(Text)
fname = Field(Text)
lname = Field(Text)
email = Field(Text)
action_date = Field(Date)
action = Field(Text)
approved = Field(Integer)
using_options(tablename = 'protocols')
def __repr__ (self):
return '%d' %(self.id)
And here is my python script:
import smtplib
import auth as auth
import ProtocolModel as PM
from elixir import *
from datetime import datetime
au = auth.UserAuth()
setup_all()
create_all()
table = PM.Protocol
records = table.query.all()
if len(records) == 0:
print 'No records in Table'
else:
for record in records:
setup_all()
if record.approved == 1:
print 'Completed'
else:
FROMADDR = au.google_user
LOGIN = FROMADDR
PASSWORD = au.google_password
TOADDRS = record.email
SUBJECT = "Weekly Research Update for " + record.irb_no + " - " + record.title
date1 = datetime.strptime(str(record.action_date), '%Y-%m-%d')
date2 = date1.strftime('%B %d, %Y')
msg = ("From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n" % (FROMADDR, ", ".join(TOADDRS), SUBJECT) )
msg += 'Dear' + ' ' + record.prefix + ' ' + record.fname + ' ' + record.lname + ',' + '\n' + '\n'
msg += 'Our records indicate that the most recent action on your research protocol titled ' + record.title + ' was taken on ' + str(date2) +'.' +'\n'
msg += 'This action was: ' + record.action
server = smtplib.SMTP('smtp.gmail.com', 587)
server.set_debuglevel(1)
server.ehlo()
server.starttls()
server.login(LOGIN, PASSWORD)
server.sendmail(FROMADDR, TOADDRS, msg)
server.quit()
This seems like it should be a relatively simple change, but I'm just not sure how to go about this. Any help is much appreciated, thanks!

I figured it out. Used ManyToOne in the new entity class in the model file and referenced that in the python code. It was a simple fix once I got back into it.
New elixir model:
from elixir import *
import auth as auth
au = auth.UserAuth()
metadata.bind = 'mysql://' + au.user + ':' + au.password + '#localhost/' + au.database
metadata.bind.echo = True
class Investigator(Entity):
id = Field(Integer, primary_key = True)
prefix = Field(Text)
fname = Field(Text)
lname = Field(Text)
email = Field(Text)
using_options(tablename = 'investigators')
def __repr__ (self):
return '%d' %(self.id)
class Protocol(Entity):
id = Field(Integer, primary_key = True)
irb_no = Field(Text)
title = Field(Text)
investigator = ManyToOne('Investigator', colname='investigator_id')
action_date = Field(Date)
action = Field(Text)
approved = Field(Integer)
using_options(tablename = 'protocols')
def __repr__ (self):
return '%d' %(self.id)
New python script:
import smtplib
import auth as auth
import ProtocolModel1 as PM
from elixir import *
from datetime import datetime
au = auth.UserAuth()
setup_all()
create_all()
table = PM.Protocol
records = table.query.all()
if len(records) == 0:
print 'No records in Table'
else:
for record in records:
setup_all()
if record.approved == 1:
print 'Completed'
else:
FROMADDR = au.google_user
LOGIN = FROMADDR
PASSWORD = au.google_password
TOADDRS = record.investigator.email
SUBJECT = "Weekly Research Update for " + record.irb_no + " - " + record.title
date1 = datetime.strptime(str(record.action_date), '%Y-%m-%d')
date2 = date1.strftime('%B %d, %Y')
msg = ("From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n" % (FROMADDR, ", ".join(TOADDRS), SUBJECT) )
msg += 'Dear' + ' ' + record.investigator.prefix + ' ' + record.investigator.fname + ' ' + record.investigator.lname + ',' + '\n' + '\n'
msg += 'Our records indicate that the most recent action on your research protocol titled ' + record.title + ' was taken on ' + str(date2) +'.' +'\n'
msg += 'This action was: ' + record.action
server = smtplib.SMTP('smtp.gmail.com', 587)
server.set_debuglevel(1)
server.ehlo()
server.starttls()
server.login(LOGIN, PASSWORD)
server.sendmail(FROMADDR, TOADDRS, msg)
server.quit()

SMTP sent mail to many recipients but doesn't received it

I've written a Python script to automatically send some information to my friends. I used SMTPlib, it works well if I only sent to me or one additional email.
When I try to send to 17 emails, (including my sender email), then it shows in sent mail on web-based Gmail. I saw that the mail was sent but I didn't receive it. Only the first recipient received the email.
If I reply to all from that mail, then everyone got only that reply.
I can't figure out why they didn't receive it when I sent it from script, I ask my friend check spam, but she didn't find anything.
This is my code:
#!/usr/bin/env python
import smtplib
import csv
from datetime import datetime, timedelta
SMTP_SERVER = 'smtp.gmail.com'
SMTP_PORT = 587
sender = 'MYBOT#gmail.com'
password = None
with open('pass', 'rt') as f:
password = f.read().strip('\n')
def send_mail(recipient, subject, body):
"""
Send happy bithday mail
"""
headers = ["From: " + sender,
"Subject: " + subject,
"To: " + recipient,
"MIME-Version: 1.0",
"Content-Type: text/html"]
headers = "\r\n".join(headers)
smtp = smtplib.SMTP(SMTP_SERVER, SMTP_PORT)
smtp.ehlo()
smtp.starttls()
smtp.ehlo
smtp.login(sender, password)
body = "" + body +""
smtp.sendmail(sender, recipient, headers + "\r\n\r\n" + body)
print "Sent to ",
print recipient
smtp.quit()
def send_happybirthday(recipient):
body = """Happy birthday to you!
\n<br/>From C2k8pro with love
"""
subject ='[BirthReminder] Happy birthday to you! from C2k8pro'
send_mail(recipient, subject, body)
def send_notification(all_mails, names):
body = """Tomorrow is birthday of %s""" % names
send_mail(all_mails, body, body)
def test_send_mail():
notify_body = """Tomorrow is birthday of """
recipients = ['MYBOT#gmail.com']
today = datetime.now()
format = "%d-%m-%Y"
print today
today_in_str = datetime.strftime(today, format)
def read_csv():
FILENAME = 'mails.csv'
reader = csv.reader(open(FILENAME, 'rt'), delimiter=',')
today = datetime.now()
one_day = timedelta(days=1)
tomorrow = today + one_day
all_mails = []
str_format = "%d/%m"
str_today = today.strftime(str_format)
str_tomorrow = tomorrow.strftime(str_format)
print 'Today is ', str_today
tomorrow_birth = []
for row in reader:
name = row[1].strip()
dob = row[2]
dmy = dob.split("/")
mail = row[3]
all_mails.append(mail)
#TODO fix dob with only 1 digit
birth_date = dmy[0] + "/" + dmy[1]
if str_today == birth_date:
print 'Happy birthday %s' % name
try:
send_happybirthday(mail)
except Exception, e:
print e
elif str_tomorrow == birth_date:
tomorrow_birth.append(name)
print "Tomorrow is %s's birthday" % name
# Remove empty string
all_mails = filter(None, all_mails)
print 'All mails: ', len(all_mails)
str_all_mails = ', '.join(all_mails)
if tomorrow_birth:
all_tomorrow = ', '.join(tomorrow_birth)
send_notification(str_all_mails, all_tomorrow)
def main():
read_csv()
if __name__ == "__main__":
main()
Can anyone explain this. Thanks!

I found solution from here
Send Email to multiple recipients from .txt file with Python smtplib
I passed a string contain all recipients separated by comma to msg['To'] and sendmail().
It's true for msg['To'] but with sendmail, I have to use a list.