I have subscribed to multiple topics using paho mqtt client. On receiving the messages from broker, I want to store the messages into mysql database. I want to gather the messages altogether before inserting into DB.I have set the threshold say 1000 messages. Only when the threshold is reached the messages has to be inserted into DB altogether at once. I am checking the row_count after cursor.execute(). But it shows the count as 1. So the bulk insert is not happening. here is my sample code snippet
//main.py
#mysql database class
db = MySQLDBClass()
#mqtt client class where subscription,connection to broker,some callbacks
mqttclient = MyMQTTClient()
mqttclient.on_message = db.onMessage
mqttclient.loop_forever()
//MySQLDBClass.py
def __init__(self):
self.insertcounter = 0
self.insertStatement = ''
self.bulkpayload = ''
self.maxInsert = 1000
def onMessage(self, client, userdata, msg):
if msg.topic.startswith("topic1/"):
self.bulkpayload += "(" + msg.payload.decode("utf-8") + "," + datetime + "),"
elif msg.topic.startswith("topic2/"):
self.insertStatement += "INSERT INTO mydatabase.table1 VALUES (" + msg.payload.decode("utf-8") + "," + datetime + ");"
elif msg.topic.startswith("topic3/")
self.insertStatement += "INSERT INTO mydatabase.table2 VALUES (" +msg.payload.decode("utf-8") + "," + datetime + ");"
elif msg.topic.startswith("messages"):
self.insertStatement += "INSERT INTO mydatabase.table3 VALUES ('" + msg.topic + "'," + msg.payload.decode("utf-8") + "," + datetime + ");"
else:
return # do not store in DB
self.insertcounter += 1
if ( self.insertcounter > self.maxInsert ):
if ( self.bulkpayload != '' ):
self.insertStatement += "INSERT INTO mydatabase.table4 VALUES" + self.bulkpayload + ";"
self.bulkpayload = ''
cursor.execute(self.insertStatement)
cursor.commit()
print (cursor.rowcount) #prints always count as one , expecting bulk count
self.insertcounter = 0
self.insertStatement = ''
with pymysql module, execute can only execute only one query at a time whereas by using mysql-connector-python, we can set multi=True in execute(mutli=true) to execute multiple statements.
https://dev.mysql.com/doc/connector-python/en/connector-python-api-mysqlcursor-execute.html
Related
I follow the price drops on the target site. If there is a price decrease in accordance with the rules I have set, it is recorded in the notificate table. From there, a telegram notification is sent through the code I created in the pipelines.py file.
Sometimes the target site discounts too many products and 200 products can come from the notificate table. I'm stuck with telegram's flood protection while sending these.
Things I've tried:
1- Rotating multiple tokens
2- add sleep time
However, I cannot say that I was successful. I'm still stuck with the flood barrier.
What I want to do here is;
No matter how many notifications come from the Notificate table, queue them and send these notifications in a way that does not exceed 20 messages per second.
How can I do that.
My pipeline.py code:
def sendnotifications(self, token):
cursor = self.cnx.cursor()
req = requests
cursor.execute("SELECT * FROM notificate WHERE token= '"+token+"'")
notifications = cursor.fetchall()
for notification in notifications:
print(notification)
productid = notification[1]
url = notification[3]
name = notification[2]
old = notification[4]
new = notification[5]
price_difference = old - new
percentage = price_difference / old
percentage_str = str("%.2f" % (percentage * 100))
message = "<b>" + name + "</b>" + "\n\n" + \
str(old) + " TL >>>> " + \
str(new) + f" TL - {percentage_str}%" + "\n\n" + \
url + "\n\n" + \
if str(old) == "1.00" or str(old) == "2.00":
message = "<b>" + name + "</b>" + "\n\n" + \
"<b>" + str(new) + " TL\n\n" + "</b>" + \
url + "\n\n" + \
token_list = [
"xxxxxxxxxxxxxxxxxxxxxxx",
"yyyyyyyyyyyyyyyyyyyyyyyy",
"zzzzzzzzzzzzzzzzzzzzzzzzzzz",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
"ccccccccccccccccccccccccccccccccc",
]
TOKEN = token_list[random.randint(0, len(token_list)-1)]
chat_id = "-100xxxxxxxxxxxxx"
bot = telegram.Bot(token=TOKEN)
# tel_url = bot.sendMessage(chat_id = chat_id, text = message, parse_mode=ParseMode.HTML)
try:
bot.sendMessage(chat_id = chat_id, text = message, parse_mode=ParseMode.HTML)
sleep(0.05)
except Exception:
return False
cursor.close()
return True
The obvious way seems to be to chunk the updates into batches of 20 and sleep for more than 1 second between them:
# ...
notifications = cursor.fetchall()
cursor.close()
for i in range(0, len(notifications), 20):
chunk = notifications[i:i+20]
for notification in chunk:
print(notification)
# ...
time.sleep(1.5)
Wrote a function that tries to reconnect to SSH when a disconnect happens. Basically expanded my existing function that simply saved the images, which works fine. The code runs but does not work to re-establish connectivity. Any help would be appreciated.
def get_image_id_and_upload_folder_of_images(db_name, table_name, selector_list,
condition_label, condition_val, img_address):
"""
get image id using project name from db
:param db_name: str - name of the data-base (usually 'server')
:param table_name: str - name of the table (usually 'images')
:param selector_list: list of str - list of selectors for the query (usually ["id", "path"])
:param condition_label: str - condition for the sql statement (like 'path')
:param condition_val: list of str - value for the condition of the condition_label (like ['name_of_file.png'])
:param img_address: str - address of the images to send them to the ftp server
:return: returns image or project id
"""
client = paramiko.client.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
retry_interval = 1
retry_interval = float(retry_interval)
timeout = int(20)
timeout_start = time.time()
while time.time() < timeout_start + timeout:
time.sleep(retry_interval)
try:
db, cur = db_info(db_name)
cond_list = ["'" + str(x) + "'" for x in condition_val]
condition = ",".join(cond_list)
selector = ",".join(selector_list)
# make a query for sql
query_stmt = "SELECT " + selector + " FROM " + table_name + " WHERE `" + \
condition_label + "` IN (" + str(condition) + ");"
image_ids = get_image_ids(cur, db, query_stmt, condition_val)
for idx, ids in enumerate(image_ids):
print(ids)
save_img_new_server(img_address + '/' + condition_val[idx], str(ids))
save_img_new_server(img_address + '/' + condition_val[idx], str(ids), hst="site.com",
folder_path='images/')
except paramiko.ssh_exception.NoValidConnectionsError as e:
print('SSH transport is not ready...')
continue
# print(img_address + '/' + condition_val[idx], str(ids))
return image_ids
Your code never calls client.connect(). In fact it doesn't interact with any paramiko module at all inside the while loop.
I'm trying to send an html table in an email. I'm not sure if this is the best way or not, but right now I read in the CSV, create a table, and then email the info.
Here is what I have so far:
def create_table():
print("<html><table>")
# column headers
print("<th>")
print("<td>Tech</td>")
print("<td>Total</td>")
print("<td>Average</td>")
print("</th>")
infile = open("test.csv","r")
for line in infile:
row = line.split(",")
tech = row[0]
total = row[1]
average = row[2]
print("<td>%s</td>" % tech)
print("<td>%s</td>" % total)
print("<td>%s</td>" % average)
print("</table></html>")
SERVER = "localhost"
FROM = "forsdani#amazon.com"
TO = ["forsdani#amazon.com"] # must be a list
SUBJECT = "this is my email"
TEXT = "TABLE GOES HERE?"
message = """\
From: %s
To: %s
Subject: %s
%s
""" % (FROM, ", ".join(TO), SUBJECT, TEXT)
# Send the mail
server = smtplib.SMTP(SERVER)
server.sendmail(FROM, TO, message)
server.quit()
If I do something like
TEXT = create_table()
then I get "None" for the text in the email. I'm not sure how to correct get the data from the function to the body of the email.
Your function does not return any values, it only prints them.
Instead of something like:
def create_table():
print("Hello, World!")
output = create_table() # output is None
Try:
def create_table():
string_to_return = "Hello, World!"
return string_to_return
output = create_table() # output is "Hello, World!"
Here's the function changed to return a value:
def create_table():
message = ""
message += "<html><table>" + "\n"
# column headers
message += "<th>" + "\n"
message += "<td>Tech</td>" + "\n"
message += "<td>Total</td>" + "\n"
message += "<td>Average</td>" + "\n"
message += "</th>" + "\n"
infile = open("test.csv","r")
for line in infile:
row = line.split(",")
tech = row[0]
total = row[1]
average = row[2]
message += ("<td>%s</td>" % tech) + "\n"
message += ("<td>%s</td>" % total) + "\n"
message += ("<td>%s</td>" % average) + "\n"
message += ("</table></html>") + "\n"
return message
I seem to be having issues querying a PYODBC connection from a Flask connection in Python. Initially I had the entire file using the same connection, but noticed when you would hit the URL for the Get request it would not return the most recent data. So i adjusted script to start and kill connection each time a get request is initiated this returns the same non updated data. I do notice that occasionally it will update the data but most times it will not.
Please assist better clarify this process.
def getChg(s, cursor, arr):
getIncResolved = "SELECT COUNT(incident.number) FROM SCHEMA.OAUSER.incident incident WHERE incident.dv_resolved_by = '" + str(s) + "' AND incident.resolved_at BETWEEN '" + str(past) + "' AND '" + str(current) + "' AND incident.dv_opened_by != '" + str(s) + "';"
getTaskResolved = "SELECT COUNT(sc_task.number) FROM SCHEMA.OAUSER.sc_task sc_task WHERE sc_task.dv_closed_by = '" + str(s) + "' AND sc_task.closed_at BETWEEN '" + str(past) + "' AND '" + str(current) + "' AND sc_task.dv_opened_by != '" + str(s) + "';"
getCallStarted = "SELECT COUNT(new_call.number) FROM SCHEMA.OAUSER.new_call new_call WHERE new_call.opened_at BETWEEN '" + str(past) + "' AND '" + str(current) + "' AND new_call.dv_opened_by = '" + str(s) + "';"
i = 0
t = 0
c = 0
cursor.execute(getIncResolved)
for row in cursor.fetchall():
i = row[0]
cursor.execute(getTaskResolved)
for row in cursor.fetchall():
t = row[0]
cursor.execute(getCallStarted)
for row in cursor.fetchall():
c = row[0]
if c > -1:
test = {'Agent': str(s), 'Calls': c, 'Inc': i, 'Task': t}
arr.append(test)
#app.route('/data',methods=['GET', 'POST'])
def api_root(): cnxn=pyodbc.connect('DSN=ServiceNow;Uid=ServiceOps;Pwd=********;',autocommit=True)
cursor = cnxn.cursor()
data = []
staffjson = request.get_json(force=True)
staff = staffjson['users']
print(staff)
del data[:]
for s in staff:
getChg(s, cursor, data)
print(data)
cnxn.close()
return json.dumps(data)
The Current variable was only being created at the initial start of the script. Therefore once it was loaded it never updated to the current time.
Created variable at each request now and is working as i expected.
I'm new at exporting data, I research all over the net but it was really hard for me to understand, can someone help me to know the basic about it.
This is my main problem: I want to download a specific data from mysql base on the date range I choose in my client, then when I click the download button, I want these data from mysql to be save in my computer together the user have the option to save it as CSV/Excel, I'm using python for my webservice. Thank you
This is my code right know in my webservice:
#api.route('/export_file/', methods=['GET', 'POST'])
def export_file():
if request.method == 'POST':
selectAttendance = """SELECT * FROM attendance"""
db.session.execute(selectAttendance)
db.session.commit()
f = csv.writer(open("file.csv", "w"))
for row in selectAttendance:
f.writerow([str(row)])
return jsonify({'success': True})
In general:
Set the mime header "Content-Type" part of the http header to the corresponding MIME-Type matching your data.
This tells the browser what type of data the webserver is going to send.
Send the actual data in the 'body'
With flask:
Forcing application/json MIME type in a view (Flask)
http://flask.pocoo.org/docs/0.10/patterns/streaming/
def get(self):
try:
os.stat(BACKUP_PATH)
except:
os.mkdir(BACKUP_PATH)
now = datetime.now() # current date and time
year = now.strftime("%Y")
month = now.strftime("%m")
day = now.strftime("%d")
time = now.strftime("%H:%M:%S")
date_time = now.strftime("%d_%m_%Y_%H:%M:%S")
TODAYBACKUPPATH = BACKUP_PATH + '/' + date_time
try:
os.stat(TODAYBACKUPPATH)
except:
os.mkdir(TODAYBACKUPPATH)
print ("checking for databases names file.")
if os.path.exists(DB_NAME):
file1 = open(DB_NAME)
multi = 1
print ("Databases file found...")
print ("Starting backup of all dbs listed in file " + DB_NAME)
else:
print ("Databases file not found...")
print ("Starting backup of database " + DB_NAME)
multi = 0
if multi:
in_file = open(DB_NAME,"r")
flength = len(in_file.readlines())
in_file.close()
p = 1
dbfile = open(DB_NAME,"r")
while p <= flength:
db = dbfile.readline() # reading database name from file
db = db[:-1] # deletes extra line
dumpcmd = "mysqldump -h " + DB_HOST + " -u " + DB_USER + " -p" + DB_USER_PASSWORD + " " + db + " > " + pipes.quote(TODAYBACKUPPATH) + "/" + db + ".sql"
os.system(dumpcmd)
gzipcmd = "gzip " + pipes.quote(TODAYBACKUPPATH) + "/" + db + ".sql"
os.system(gzipcmd)
p = p + 1
dbfile.close()
else:
db = DB_NAME
dumpcmd = "mysqldump -h " + DB_HOST + " -u " + DB_USER + " -p" + DB_USER_PASSWORD + " " + db + " > " + pipes.quote(TODAYBACKUPPATH) + "/" + db + ".sql"
os.system(dumpcmd)
gzipcmd = "gzip " + pipes.quote(TODAYBACKUPPATH) + "/" + db + ".sql"
os.system(gzipcmd)
# t = ("Your backups have been created in '" + TODAYBACKUPPATH + "' directory")
return "Your Folder have been created in '" + TODAYBACKUPPATH + "'."