I have the below Python script and it works very well, but I would like to introduce some fail safe options .. That fail safe options being ..
1) if I cannot find (in this example) Michael I would like to write to the file ERROR ..
2) If the database does not allow me to connect for whatever reason I would like to write to another file CONNECTION_ERROR
Here is my script:
#! /usr/bin/python
import pymssql
import sys
sys.path.insert(0, '/opt/mount/safe')
from secrets import password
conn = pymssql.connect(
server="server",
port=port,
user="user",
password=password,
database="database")
conn
cursor = conn.cursor()
cursor.execute("SELECT name, address FROM database WHERE name = 'michael'")
with open('safe/file.txt', 'w') as f:
for row in cursor.fetchall():
print ( "Person " + (row[0])),
print ( "has this address " + (row[1]))
f.write(str( "Person " + (row[0])))
f.write("%s\n" % str( " has this address " + (row[1])))
conn.close()
Took me a while but the below works really really well
import sys, pymssql, string, os, calendar, datetime, traceback, socket, platform
try:
d = datetime.datetime.now()
log = open("LogFile.txt","a")
log.write("----------------------------" + "\n")
log.write("----------------------------" + "\n")
log.write("Log: " + str(d) + "\n")
log.write("\n")
# Start process...
starttime = datetime.datetime.now()
log.write("Begin process:\n")
log.write(" Process started at "
+ str(starttime) + "\n")
log.write("\n")
xxxxxx
CODE HERE
XXXXXX
endtime = datetime.datetime.now()
# Process Completed...
log.write(" Completed successfully in "
+ str(endtime - starttime) + "\n")
log.write("\n")
log.close()
except:
tb = sys.exc_info()[2]
tbinfo = traceback.format_tb(tb)[0]
# Concatenate information together concerning
# the error into a message string
pymsg = "PYTHON ERRORS:\nTraceback info:\n" + tbinfo + "\nError Info:\n" + str(sys.exc_info())
# Return python error messages for use in
# script tool or Python Window
log.write("" + pymsg + "\n")
log.close()
Related
Hello I am running a script where we get data from certain websites and put them in a database using scrapy, the script runs well but in log text file it shows ''DEBUG: Starting new HTTPS connection (1): 1.rome.api.flipkart.com:443''
what can be the real problem here, any help is appreciated, here is the code
import os
import sys, getopt
import time
import datetime
import pytz
import mysql.connector
import configparser
import shutil
import time
import concurrent.futures
import pandas as pd
currentdir = os.path.dirname(os.path.realpath(__file__))
parentdir = os.path.dirname(currentdir)
sys.path.append(parentdir)
from datetime import datetime
from scrapy.utils.project import get_project_settings
from scrapy.crawler import CrawlerProcess
from multiprocessing import Process
from db.db_action import DBAction
from utils import utils
from concurrent.futures import ThreadPoolExecutor, as_completed
tz = pytz.timezone("Asia/Kolkata")
crawl_inputs = dict()
crawl_inputs["env"] = "prod"
crawl_inputs["marketplace"] = "Amazon"
crawl_inputs["site"] = "amz_kw"
crawl_inputs["db_name"] = "asian_paints"
crawl_inputs["pf_id"] = "1"
crawl_inputs["location_search"] = "0"
crawl_inputs["limit"] = ""
crawl_inputs["page"] = 1
crawl_inputs["kw_snapshot"] = 0
crawl_inputs["pdp_snapshot"] = 0
crawl_inputs["quick_search"] = 1
crawl_inputs["client_id"] = 1241
crawl_inputs["client"] = "asian_paints"
crawl_inputs["start_time"] = datetime.now(tz).strftime("%Y-%m-%d %H:%M:%S")
db_action = DBAction()
actual_count = 0
result_count = 0
archived_path = "C:/archives/"
connection = None
cursor = None
# Directory Create
try:
if not os.path.exists(archived_path):
os.makedirs(archived_path)
if not os.path.exists(archived_path + datetime.now(tz).strftime("%Y%m%d")+"/"+ crawl_inputs["site"]):
os.makedirs(archived_path + datetime.now(tz).strftime("%Y%m%d")+"/"+ crawl_inputs["site"])
if not os.path.exists("C:/var/logs/" + crawl_inputs["site"]):
os.makedirs("C:/var/logs/" + crawl_inputs["site"])
shutil.move("C:/var/logs/" + crawl_inputs["site"], archived_path + datetime.now(tz).strftime("%Y%m%d"), copy_function = shutil.copytree)
except Exception as e:
print(e)
print("File creation error: {0}:{1}".format(e.errno, e.strerror))
try:
if os.name == "nt":
log_path = "C:/var/logs"
base_dir = log_path+"/"+crawl_inputs["site"]
if not os.path.exists(base_dir):
os.makedirs(base_dir)
else:
log_path = "/var/logs/"
base_dir = log_path+"/"+crawl_inputs["site"]
if not os.path.exists(base_dir):
os.makedirs(base_dir)
directories = ["output", "run_log", "webpages"]
for directory in directories:
if not os.path.exists(base_dir+"/"+directory):
os.makedirs(base_dir+"/"+directory)
except OSError as oserr:
print("OS error occurred trying to open. Aborting.. Error{0}:{1}".format(oserr.errno, oserr.strerror))
sys.exit(1)
except IOError as ioerr:
print("I/O Error{0}: {1}".format(ioerr.errno, ioerr.strerror))
sys.exit(1)
except FileNotFoundError as fnfe:
print("File not found. Aborting.. Error: {0}:{1}".format(fnfe.errno, fnfe.strerror))
sys.exit(1)
except Exception as e:
print("File creation Error. Aborting.. Error: {0}:{1}".format(e.errno, e.strerror))
sys.exit(1)
crawl_inputs = db_action.get_kw_platform_inputs(crawl_inputs)
print(f"Total Executing Inputs : {len(crawl_inputs['inputs'])}")
print("Crawl ID: {0}".format(crawl_inputs["crawl_id"]))
def start(input):
pf_id = str(input["pf_id"])
keyword = str(input["keyword"])
brand_id = str(input["brand_id"])
brand_name = str(input["brand_name"])
keyword_id = str(input["keyword_id"])
location_id = str(input["location_id"])
location = str(input["location"])
pincode = str(input["pincode"])
location_search = str(crawl_inputs["location_search"])
env = str(crawl_inputs["env"])
db_name = str(crawl_inputs["db_name"])
crawl_id = str(crawl_inputs["crawl_id"])
site = str(crawl_inputs["site"])
page = str(crawl_inputs["page"])
command = 'python ' +currentdir+ '/main_kw.py --env="' +env+ '" --db_name="' +db_name+ '" --crawl_id="' +crawl_id+ '" --site="' +site+ '" --pf_id="'+pf_id+ '" --brand_id="' +brand_id+ '" --brand_name="' +brand_name+ '" --keyword_id="' +keyword_id+ '" --keyword="' +keyword+ '" --location_id="' +location_id+ '" --location="' +location+ '" --pincode="' +str(pincode)+ '" --page="' +page+ '" --location_search="' +location_search+ '"'
print("Executing Input :{0}".format(command))
os.system(command)
def runner():
threads = []
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
for input in crawl_inputs.get("inputs"):
print(f"Input: {input}")
task = executor.submit(start, input)
threads.append(task)
for task in concurrent.futures.as_completed(threads):
print(task.result())
runner()
time.sleep(5)
crawl_inputs["finish_time"] = datetime.now(tz).strftime("%Y-%m-%d %H:%M:%S")
connection = db_action.db_connection(db_name=crawl_inputs["db_name"], env=crawl_inputs["env"])
cursor = connection.cursor()
try:
cursor.execute("update `amazon_crawl_kw` set `status` = 0 where `crawl_id` != " +str(crawl_inputs["crawl_id"])+ ";")
cursor.execute("select count(kw_crawl_data_id) as product_count from `amazon_crawl_kw` where status = 1 and pf_id = "+str(crawl_inputs["pf_id"])+ " and crawl_id=" +str(crawl_inputs["crawl_id"])+ ";")
row = cursor.fetchone()
print("row value: "+ str(row))
result_count = row["product_count"]
print("Crawled row count : " + str(result_count))
try:
sql2 = 'UPDATE rb_crawl SET status=1, end_time = "' +str(crawl_inputs["finish_time"])+ '" ,no_of_sku_parsed='+ str(result_count)+ ' WHERE crawl_id=' + str(crawl_inputs["crawl_id"])
cursor.execute(sql2)
except Exception as e:
print("The following exception occured while updating : "+ str(e))
sql3 = 'UPDATE rb_platform SET kw_crawl_data_date = "'+ str(crawl_inputs["start_time"]) + '", kw_crawl_data_id = ' +str(crawl_inputs["crawl_id"])+ ' WHERE pf_id = ' + str(crawl_inputs["pf_id"])
cursor.execute(sql3)
connection.commit()
connection.close()
print("Updated rb_platform successfully")
except Exception as e:
print("Updating crawl id failed with exception as :" + str(e))
# try:
# items_count = result_count
# subject = crawl_inputs["site"] +" crawling completed"
# body = "Hi Team,<br><br>" +crawl_inputs["site"]+ " crawling successfully completed for the plateform " +crawl_inputs["marketplace"]+ "...<br>Platform Id: " +str(crawl_inputs["pf_id"])+ "<br>Crawl Id: " +str(crawl_inputs["crawl_id"])+ "<br>Total crawled items: " +str(items_count)+ " <br>Total Actual Items: " + str(actual_count) +" <br>Please QC the data value..<br><br>Thanks<br>Trailytics Team"
# utils.send_mail("no-reply#trailytics.com", "vijay.kothawar#trailytics.com;ashish.rawat#trailytics.com;anirudh.varshney#trailytics.com;ashutosh.shukla#trailytics.com", subject, body)
# print("Crawling process has been completed")
# except Exception as e:
# print("Mail Sending error:" + str(e))
print("Finish")
I am running an impdp statement form inside a Python File and I want to write the print statements in Python script in the logfile, which is being created in 'abccmd' using '>>' append symbol.
Two issues to be resolved:
The print commands are overwriting the logs generated using impdp commands
I want the logs to be in order in which they come in script (All the print commands are coming at top of the log file. Even the commands that are after the impdp statement in the function being called)
Also, I am using dynamic naming system for logfiles that are created.
def import_command_run(DP_WORKLOAD, dp_workload_cmd, imp_loop, vardate):
abccmd = 'impdp admin/DP_PDB_ADMIN_PASSWORD#DP_PDB_FULL_NAME SCHEMAS=ABC >>' + logdir + '/DP_IMP_' + DP_PDB_FULL_NAME[i] + '_' + DP_WORKLOAD + '_' + str(vardate) + '.log 2>&1'
defcmd = 'impdp admin/DP_PDB_ADMIN_PASSWORD#DP_PDB_FULL_NAME SCHEMAS=DEF >>' + logdir + '/DP_IMP_' + DP_PDB_FULL_NAME[i] + '_' + DP_WORKLOAD + '_' + str(vardate) + '.log 2>&1'
# any of the above commands
run_imp_cmd(eval(abccmd))
def run_imp_cmd(cmd):
output = subprocess.Popen([cmd], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
stdout,stderr = output.communicate()
return stdout
All the Print commands used in imp_workload() and the main() methods should come in exact order in the log file.
def imp_workload(DP_WORKLOAD, DP_DURATION_SECONDS, vardate):
imp_loop = 1
while DP_DURATION_SECONDS > 0:
print("------------------------------------------------------------------------------------------------------------------------------------------")
print(" PDB " + DP_PDB_FULL_NAME[i] + " for Workload " + DP_WORKLOAD + ": Import number " + str(imp_loop) + " Starts")
print("------------------------------------------------------------------------------------------------------------------------------------------")
duration = 0
print("\nImport is running for Time loop : " + str(imp_loop))
startImport = datetime.now()
start = time.time()
print("Start Time for the Import is : " + startImport.strftime("%Y-%m-%d %H:%M:%S"))
# Calling the above function here
import_command_run(DP_WORKLOAD, dp_workload_cmd, imp_loop, vardate)
time.sleep(60)
stop = time.time()
endImport = datetime.now()
print("Stop Time for the Import is : " + endImport.strftime("%Y-%m-%d %H:%M:%S"))
duration = stop - start
print("\nTotal Time elapsed for Data Pump Import Time loop " + str(imp_loop) + ": " + str(int(duration/3600)) + " hours " + str(int(((duration/60)%60))) + " minutes " + str(int((duration % 60))) + " seconds\n")
DP_DURATION_SECONDS = DP_DURATION_SECONDS - duration
if DP_DURATION_SECONDS>0:
print("\nData Pump Import will again run for: " + str(int(DP_DURATION_SECONDS)) + " seconds\n")
else:
print("\nDATA Pump Import has ended for the workload: " + DP_WORKLOAD + "\n")
imp_loop = imp_loop + 1
I am trying to use sys.stdout as you can see. But it is overwriting the logfile created by the impdp statement.
if __name__ == "__main__":
vardate = datetime.now().strftime("%d-%b-%Y-%I_%M_%S_%p")
# Running loop through each workload mentioned in the config file
for i in range((len(DP_PDB_FULL_NAME))):
print("\n==========================================================================================================================================")
print("\n Data Pump Workload has Started \n")
print("==========================================================================================================================================")
# Conversion of time form minutes to seconds
DP_DURATION_SECONDS = int(DP_DURATION) * 60
now = datetime.now()
print("Current Date and Time: " + now.strftime("%Y-%m-%d %H:%M:%S"))
print("\nData Pump Import will run for " + DP_DURATION + " minutes\n")
for DP_WORKLOAD in DP_WORKLOAD_NAME:
sys.stdout = open(logdir + '/DP_IMP_' + DP_PDB_FULL_NAME[i] + '_' + DP_WORKLOAD + '_' + str(vardate) + '.log', 'w')
p1 = multiprocessing.Process(target=imp_workload, args=(DP_WORKLOAD, DP_DURATION_SECONDS, vardate, ))
p1.start()
sys.stdout.close()
Please suggest a way to log them in a proper way.
I wonder this may be the result you want, anyway, you can try this one:
python -m abc.py > d:\output.txt
It makes output.txt in d: and it will record all the print results in the order they are called.
I am performing a daily database dump in a python script and I am looking for the most python efficient way to delete the current file only after the recent one was written successfully, i.e. delete the backup-12-11-2019.sql only after backup-13-11-2019 was created successfully
you can use try:...except:...esle: as the following code.
import datetime
import os
now = datetime.datetime.utcnow()
try:
pg_dumpall('-h', DB_HOST, '-U', DB_USERNAME, '-p', DB_PORT, _out=BACKUP_FOLDER + 'backup-' + str(now.day) + '-' + str(now.month) + '-' + str(now.year) + '.sql')
except Exception as e:
print(e)
else:
previous_day = now - datetime.timedelta(days=1)
os.remove(BACKUP_FOLDER + 'backup-' + str(now.day - 1) + '-' + str(now.month) + '-' + str(now.year) + '.sql')
If the pg_dumpall does not raise and Exception it will delete the previous backup file
Best regard
From DBA StackExchange
pg_dumpall -U pg_user > /tmp/tmp.txt
DUMP_STATUS=$?
echo "Dump status: $DUMP_STATUS" > /tmp/status.txt
And SO: Get return value
subprocess.check_output([SCRIPT, "-d", date], shell=True).
We're able to come up with something that will run the command you want to run, and check its return value at the same time.
output = subprocess.check_output(['pg_dumpall', '-h', DB_HOST, '-U', DB_USERNAME, '-p', DB_PORT], stdout=outfile)
if output == 0:
print("Success")
os.remove(oldfile)
else:
print("Failure")
I have a configuration file I'm reading with ConfigParser.
One of the values is a file path containing spaces in the path:
[Configuration]
mysqldumpexecutable_location = C:/Program Files (x86)/MySQL/MySQL Server 5.7/bin/
It seems that the value returned with get() for this parameter is only "C:/Program" - up to the space.
Here's the relevant code:
import os
import time
import datetime
import configparser
def BackupDB():
try:
config = configparser.RawConfigParser()
config.read(r"etc\configuration.txt")
DB_HOST = config.get('Configuration', 'db_host')
DB_USER = config.get('Configuration', 'db_username')
DB_USER_PASSWORD = config.get('Configuration', 'db_password')
#DB_NAME = '/backup/dbnames.txt'
DB_NAME = config.get('Configuration', 'db_schema')
BACKUP_PATH = config.get('Configuration', 'backup_path')
MYSQLDUMP_LOCATION = config.get('Configuration', 'mysqldumpexecutable_location')
DATETIME = time.strftime('%d%m%Y-%H%M%S')
TODAYBACKUPPATH = BACKUP_PATH + DATETIME
db = DB_NAME
dumpcmd = MYSQLDUMP_LOCATION + "mysqldump -u " + DB_USER + " -p" + DB_USER_PASSWORD + " " + db + " > " + TODAYBACKUPPATH + "/" + db + ".sql"
os.system(dumpcmd)
except Exception as e:
print("something here")
return None
I'm getting this error:
'c:/Program' is not recognized as an internal or external command,
operable program or batch file.
How can I pass this Windows path correctly?
Thanks!
I'm new at exporting data, I research all over the net but it was really hard for me to understand, can someone help me to know the basic about it.
This is my main problem: I want to download a specific data from mysql base on the date range I choose in my client, then when I click the download button, I want these data from mysql to be save in my computer together the user have the option to save it as CSV/Excel, I'm using python for my webservice. Thank you
This is my code right know in my webservice:
#api.route('/export_file/', methods=['GET', 'POST'])
def export_file():
if request.method == 'POST':
selectAttendance = """SELECT * FROM attendance"""
db.session.execute(selectAttendance)
db.session.commit()
f = csv.writer(open("file.csv", "w"))
for row in selectAttendance:
f.writerow([str(row)])
return jsonify({'success': True})
In general:
Set the mime header "Content-Type" part of the http header to the corresponding MIME-Type matching your data.
This tells the browser what type of data the webserver is going to send.
Send the actual data in the 'body'
With flask:
Forcing application/json MIME type in a view (Flask)
http://flask.pocoo.org/docs/0.10/patterns/streaming/
def get(self):
try:
os.stat(BACKUP_PATH)
except:
os.mkdir(BACKUP_PATH)
now = datetime.now() # current date and time
year = now.strftime("%Y")
month = now.strftime("%m")
day = now.strftime("%d")
time = now.strftime("%H:%M:%S")
date_time = now.strftime("%d_%m_%Y_%H:%M:%S")
TODAYBACKUPPATH = BACKUP_PATH + '/' + date_time
try:
os.stat(TODAYBACKUPPATH)
except:
os.mkdir(TODAYBACKUPPATH)
print ("checking for databases names file.")
if os.path.exists(DB_NAME):
file1 = open(DB_NAME)
multi = 1
print ("Databases file found...")
print ("Starting backup of all dbs listed in file " + DB_NAME)
else:
print ("Databases file not found...")
print ("Starting backup of database " + DB_NAME)
multi = 0
if multi:
in_file = open(DB_NAME,"r")
flength = len(in_file.readlines())
in_file.close()
p = 1
dbfile = open(DB_NAME,"r")
while p <= flength:
db = dbfile.readline() # reading database name from file
db = db[:-1] # deletes extra line
dumpcmd = "mysqldump -h " + DB_HOST + " -u " + DB_USER + " -p" + DB_USER_PASSWORD + " " + db + " > " + pipes.quote(TODAYBACKUPPATH) + "/" + db + ".sql"
os.system(dumpcmd)
gzipcmd = "gzip " + pipes.quote(TODAYBACKUPPATH) + "/" + db + ".sql"
os.system(gzipcmd)
p = p + 1
dbfile.close()
else:
db = DB_NAME
dumpcmd = "mysqldump -h " + DB_HOST + " -u " + DB_USER + " -p" + DB_USER_PASSWORD + " " + db + " > " + pipes.quote(TODAYBACKUPPATH) + "/" + db + ".sql"
os.system(dumpcmd)
gzipcmd = "gzip " + pipes.quote(TODAYBACKUPPATH) + "/" + db + ".sql"
os.system(gzipcmd)
# t = ("Your backups have been created in '" + TODAYBACKUPPATH + "' directory")
return "Your Folder have been created in '" + TODAYBACKUPPATH + "'."