We Keep Coding

AttributeError: 'ChangeDetectorService' object has no attribute 'process'

I'm trying to develop Job program which in turn calls Service program.
ChangeDetectorService does have procedure named process. I don't know why I get the warning message:
AttributeError: 'ChangeDetectorService' object has no attribute 'process'
"change_detector_job.py" and ChangeDetectorService source code are attached.
# change_detector_job.py
import argparse
import datetime
# import multiprocessing
import os
# import sys
# import traceback
import sys
import traceback
from common.utils.config import Config, log_start, log_end
from ReplicationService.module.ChangeDetectorService import ChangeDetectorService
args = None
main_config = None
main_logger = None
app_name = os.path.basename(sys.argv[0]).replace('.py', '') + '_main'
class ChangeDetectorJob:
def __init__(self):
self.config = None
self.logger = None
self.env = 'test'
def setup(self):
self.env = args.env
self.config = Config(args.config, app_name)
self.logger = self.config.get_logger()
def process(self):
self.setup()
t1 = log_start("ChangeDetector job started (UTC): %s" % (datetime.datetime.utcnow()), with_memory=True,
logger=self.logger)
service = ChangeDetectorService(args.config, app_name)
success = service.process()
log_end("ChangeDetector job completed (UTC): %s" % (datetime.datetime.utcnow()), success=success, start_time=t1,
logger=self.logger)
if __name__ == '__main__':
try:
parser = argparse.ArgumentParser(description="Data ChangeDetector Job")
parser.add_argument("-e", "--env", default="test", choices=['test', 'dev', 'uat', 'prd'],
help="environment this job to be deployed (default %default)")
parser.add_argument("-c", "--config", default="../config/cmo_backend_local_config.ini",
help="config file (test only default %default)")
args = parser.parse_args()
ini_file = args.config
main_config = Config(ini_file, app_name)
main_logger = main_config.get_logger()
# multiprocessing.set_start_method('spawn')
main_logger.info("**** ChangeDetector job started (UTC): %s" % datetime.datetime.utcnow())
# service = ChangeDetectorService(args.config, app_name)
# success = service.process()
ChangeDetectorJob().process()
main_logger.info("**** ChangeDetector job completed (UTC): %s" % datetime.datetime.utcnow())
except Exception as error:
if main_logger is not None:
main_logger.error("Exception encountered: " + str(error))
main_logger.error(traceback.format_exc())
traceback.print_exc()
print("FINAL EXCEPTION OUT .... ")
# ChangeDetectorService.py
# import mysql.connector
from datetime import datetime
from datetime import timedelta
from common.database.mysql_manager import MysqlManager
from common.utils.config import Config, log_start, log_end
# mysql.connector.connect(host='localhost',port="3306",user='user',password='pass',database='dbname')
import mysql.connector
# rows_inserted = 0;
class ChangeDetectorService:
#
def __init__(self, ini_file, app_name):
self.config_object = Config(ini_file, app_name)
self.logger = self.config_object.get_logger()
self.mysql_manager = MysqlManager(self.config_object)
try:
def get_connection(self):`enter code here`
connection = mysql.connector.connect(user='user', password='zzzz',
host='host',
database='dbname'
)
return connection
def init_change_detector(self):
print("Change Detector started at " + datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
mysqlManager = MysqlManager
# conn = self.get_connection()
conn = self.mysqlManager.get_db_connection()
cursor = conn.cursor()
query = ('SELECT table_name, column_name, key_name '
'FROM cmooptimizationsvc.csj_chngdtct_tab_col '
'WHERE is_active=1 '
'ORDER BY table_name, column_name')
cursor.execute(query)
# get all records
records = cursor.fetchall()
for record in records:
self.process_col_tab_chg(record[0], record[1], record[2])
if conn.is_connected():
conn.close()
cursor.close()
def insert_change_log(self, table_name, key_name, attr_name, old_attr_value, new_attr_value):
# global rows_inserted;
insert_query = """INSERT INTO csj_shipment_changelog(table_name, key_name,
attr_name, old_attr_value,
new_attr_value)
VALUES (%s, %s, %s, %s, %s)"""
conn = self.get_connection()
cursor = conn.cursor()
tuple1 = (table_name, key_name, attr_name, old_attr_value, new_attr_value)
# tuples.append(tuple1)
cursor.execute(insert_query, tuple1)
# rows_inserted += 1
# print( rows_inserted );
# if (rows_inserted%10==0):
# cursor.executemany(insert_query, tuples)
conn.commit()
rows_inserted = 0
# tuples = []
# quit()
cursor.close()
conn.close()
# Look for Shipment, in past date
def find_past_shipment(self,
table_name,
key_name,
column_name,
before_date,
curr_key
):
saved_col_name = column_name
saved_key_name = key_name
conn = self.get_connection()
cursor = conn.cursor()
query = 'SELECT ' + saved_key_name + ' , ' + saved_col_name + ' FROM ' + table_name \
+ ' where rec_cre_dt_utc < ' + "'" + before_date.strftime('%Y-%m-%d 00:00:00') + "'" \
+ ' and shipment_num = ' + "'" + curr_key + "'" + ' order by rec_cre_dt_utc desc LIMIT 1'
cursor.execute(query)
records = cursor.fetchone()
cursor.close()
conn.close()
if records is not None:
past_attr_val = records[1]
return past_attr_val
else:
return 0
def process_col_tab_chg(self,table_name, column_name, key_name):
saved_key_name = key_name
saved_col_name = column_name
old_val = 0
ini_time_for_now = datetime.now()
date_before_1day = ini_time_for_now - timedelta(days=1)
query = 'SELECT ' + key_name + ' , ' + saved_col_name + ' , ' + ' rec_cre_dt_utc FROM ' + table_name \
+ ' where rec_cre_dt_utc >= ' + "'" + date_before_1day.strftime('%Y-%m-%d 00:00:00') + "'"
conn = self.get_connection()
cursor = conn.cursor()
cursor.execute(query)
for (key_name, column_name, rec_cre_dt_utc) in cursor:
curr_attr_val = column_name
curr_key_val = key_name
old_val = self.find_past_shipment(table_name,
saved_key_name,
saved_col_name,
rec_cre_dt_utc,
curr_key_val
)
if curr_attr_val != old_val \
and old_val != 0:
self.insert_change_log(table_name, key_name, saved_col_name, old_val, curr_attr_val )
else:
continue
cursor.close
conn.close()
def cleanup():
print("Change Detector stopped " + datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
def process():
start = datetime.now()
init_change_detector()
# cleanup()
# mysqlManager = MysqlManager
# conn = get_connection(self)
# conn = self.mysql_manager.get_db_connection()
# cursor = conn.cursor()
# query = ('SELECT table_name, column_name, key_name '
# 'FROM cmooptimizationsvc.csj_chngdtct_tab_col '
# 'WHERE is_active=1 '
# 'ORDER BY table_name, column_name')
# cursor.execute(query)
# # get all records
# records = cursor.fetchall()
# for record in records:
# self.process_col_tab_chg(record[0], record[1], record[2])
# if conn.is_connected():
# conn.close()
# cursor.close()
end = datetime.now()
time_diff = (end - start)
execution_time = time_diff.total_seconds()
print("Elapsed time(secs): " + str(execution_time))
except Exception as e:
print("Exception " + e)
finally:
cleanup()
# if __name__ == "__main__":
# main()

You get AttributeError when you try to use an attribute (method or property) which does not exist for your object (class). In this case, the error states that ChangeDetectorService is under the question:
AttributeError: 'ChangeDetectorService' object has no attribute 'process'
In your error message, right before the above line, you should see the code line which caused this error:
success = service.process()
service was assigned to this expression:
service = ChangeDetectorService(args.config, app_name)
So, service is of type (class) ChangeDetectorService. It must be it... Inspecting the source code we see this:
class ChangeDetectorService:
#
def __init__(self, ini_file, app_name):
self.config_object = Config(ini_file, app_name)
self.logger = self.config_object.get_logger()
self.mysql_manager = MysqlManager(self.config_object)
try:
...
try has the same indentation as class. So, try has closed the ChangeDetectorService class definition. It means, your ChangeDetectorService class has only the constructor (__init__) defined. ChangeDetectorService class does not have any other method defined. It means, it doesn't have process method either. That's why you get the error. If you want ChangeDetectorService class to have the process method, make sure try or anything else does not interfere with all the method definitions of the class.

How can i run all my code in one function

My python code doesnt work. I get an output for only success mysql connection.
I want to print group id, hostname and other variables. The only output i get is
('Connected to MySQL Server version ', u'5.7.36-0ubuntu0.18.04.1')
("You're connected to database: ")
I cannot print group id or anything else. Im a newbie in python :(
import os
import mysql.connector
import json
execfile("/home/manager/test/mysqlconnector.py")
active_ip = ""
hostname = ""
group_id = 0
def my_funciton():
query = "select value_oid from snmp_trap where name_oid = '1.3.6.1.4.1.14823.2.3.3.1.200.1.17.0'"
cursor = connection.cursor(dictionary=True)
cursor.execute(query)
mac = cursor.fetchone()
mac_string = mac.values()
mac_str = json.dumps(mac_string)
mac_ = mac_str.replace(':','')
mac_ = mac_.replace('"','')
mac_ = mac_.replace(']','')
mac_ = mac_.replace('[','')
return mac_
active_mac = my_function()
query = "select epp_active_ip, epp_hostname, epp_group_id from epp_inventory where epp_active_mac = + 'active_mac.upper()'"
cursor = connection.cursor(dictionary=True)
cursor.execute(query)
rows = cursor.fetchall()
#active_ip = ""
#hostname = ""
#group_id = 0
for row in rows:
active_ip = row["epp_active_ip"]
hostname = row["epp_hostname"]
group_id = row["epp_group_id"]
print(group_id)
query = "select wmic_id from group_wmic where group_id = " + str(group_id)
cursor = connection.cursor(dictionary=True)
cursor.execute(query)
wmic_ids = cursor.fetchall()
for row in wmic_ids:
query = "select command_line from wmic_commands where id = " + row["wmic_id"]
cursor = connection.cursor(dictionary=True)
cursor.execute(query)
command_line = cursor.fetchone()
os.system(command_line)
os.system("ls -al")
#os.system(command)
my_funciton()

Apart from naming and indentation issues, which you should really fix, because it will make your code a nightmare to maintain - the issue is quite simple:
Consider:
def some_function():
print('this prints')
return
print('this does not')
Your code has the exact same problem. In your function my_funciton, you have the following line:
return mac_
Nothing after that will ever execute. You need to put the return statement in the position of the function's code where you expect it to actually return. You cannot put it just anywhere and expect the function to execute the rest of the code.

Django select_for_update function in concurrent process

When I testing the function of update_or_create in multi threading condition, I found the result is not what I wanted,they created more than one record in MySQL. As the code show, update_or_create used select .. for update to lock rows in MySQL, then it should be only one record in MySQL. I used SQLAlchemy and row sql has proved that.
So, is the Django codes wrong?
with Django code:
def get_or_create_ins():
p, created = OxalicAcid.objects.update_or_create(defaults={"formula": "20", "degree": "80"}, name="smart")
def run():
for i in range(10):
t = threading.Thread(target=get_or_create_ins, args=())
t.start()
if __name__ == "__main__":
# more than one record will be created
run()
with SQLAlchemy code:
#contextmanager
def transaction_atomic():
session = Session()
try:
yield session
session.commit()
except Exception as e:
session.rollback()
raise e
def get_result_with_update(session, name):
sql = text("""
select * from acid_oxalicacid where name = :name for update
""")
params = dict(name=name)
cursor = session.execute(sql, params)
result = cursor.fetchall()
return result
def get_result(session, name):
sql = text("""
select * from acid_oxalicacid where name = :name
""")
params = dict(name=name)
cursor = session.execute(sql, params)
result = cursor.fetchall()
return result
def create_data(session, name, degree, formula):
sql = text("""
insert into acid_oxalicacid (name, degree, formula) values (:name, :degree, :formula)
""")
params = dict(
name=name,
degree=degree,
formula=formula
)
session.execute(sql, params)
def get_or_create():
name = "smart"
degree = "50"
formula = "100"
with transaction_atomic() as session:
res = get_result_with_update(session, name)
if not res:
create_data(session, name, degree, formula)
res = get_result(session, name)
return res
if __name__ == "__main__":
# Only one record be created, that's correct
for i in range(10):
t = threading.Thread(target=get_or_create, args=())
t.start()

Because Django use the 'read committed' transaction isolation level, so it will be multiply records, if change it to 'repeatable read', it's will be only one record in database.

nginx uwsgi bottle postgresql psycopg2

My requirement:
About 300 (field sensor-like) clients report their status (json string) once every X (say 10) minutes. They can only use "curl" to do the reporting . The API/handler on the server needs to parse and dump all those values into the master table.
A few users (usually less than 10) connect to the app from their browsers to check the status of the sensor-reports and may linger around checking a few pages (main status page, detailed report page etc.). There are less than 10 pages (think reports) that the users want to see.
My setup:
Web Server: nginx
App Server: uwsgi
Framework: Bottle
Database: PostgreSQL
Python DB Driver: psycopg2
Frontend: Bootstrap
My code:
Please note that I did not include a lot of error checking and other security measures that we have in code, simply because they do not contribute to this discussion.
import os
from bottle import route, post, run, request, template, install, static_file
import psycopg2
import customemailservice
#route('/static/<filepath:path>')
def server_static(filepath):
return static_file(filepath, root='/webapp/ss/static')
# The URL that the sensors will hit to pass on their status report
#post('/logger')
def dolog():
sensor_id = request.json['id']
sensor_ts = request.json['ts']
sensor_ut = request.json['main']['ut']
sensor_ploss = request.json['main']['ploss']
sensor_s1 = request.json['main']['s1']
sensor_s2 = request.json['main']['s2']
sensor2_status = request.json['aux']['status']
sensor2_rts = request.json['aux']['rts']
try:
conn = psycopg2.connect('dbname=<dbnane> user=<username> password= <password> host=<dbhost> port=<dbport>')
except psycopg2.Error as pe:
print pe.pgerror
curr = conn.cursor()
if conn != None and curr != None:
curr.execute('''INSERT INTO tbllog (id, ts, ut, ploss, s1, s2, status, rts) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)''',(sensor_id, sensor_ts, sensor_ut, sensor_ploss, sensor_s1, sensor_s2, sensor2_status, sensor2_rts))
conn.commit()
curr.close()
conn.close()
else:
pass
# The code here is irrelevant
return template('Ok from {{sid}} at {{ts}}', sid=sid,ts=ts)
#route('/')
def index():
try:
conn = psycopg2.connect('dbname=<dbnane> user=<username> password=<password> host=<dbhost> port=<dbport>')
except psycopg2.Error as pe:
conn = None
print pe.pgerror
curr = conn.cursor()
if conn != None and curr != None:
sql = 'select t1.* from tbllog t1 where t1.ts = (select max(t2.ts) from tbllog t2 where t2.id=t1.id) order by id;'
curr.execute(sql)
rs=curr.fetchall()
html=""
for row in rs:
html = html + '<tr><td class="warning">' + row[0] + '</td><td class="warning">' + str(row[1]) + '</td><td class="success">' + str(row[2]) + '</td><td class="success">' + str(row[3]) + '</td><td class="success">' + str(row[4]) + '</td><td class="info">' + str(row[5]) + '</td><td class="info">' + str(row[6]) + '</td></tr>'
curr.close()
conn.close()
# Pass the raw html table that will be inserted into the index template.
return template('index',tdata=html)
#route('/status/<sensor_id>')
def getsensorid(sensor_id):
try:
conn = psycopg2.connect('dbname=<dbnane> user=<username> password=<password> host=<dbhost> port=<dbport>')
except psycopg2.Error as pe:
conn = None
print pe.pgerror
curr = conn.cursor()
if conn != None and curr != None:
sql = 'select * from tbllog where id=\'' + sensor_id + '\' order by ts;'
curr.execute(sql)
rs=curr.fetchall()
html=""
for row in rs:
html = html + '<tr class="info"><td>' + row[0] + '</td><td>' + str(row[1]) + '</td><td>' + str(row[2]) + '</td><td>' + str(row[3]) + '</td><td>' + str(row[4]) + '</td><td>' + str(row[5]) + '</td><td>' + str(row[6]) + '</td></tr>'
curr.close()
conn.close()
if __name__ == '__main__':
run (host="0.0.0.0", port=8080, debug=True)
else:
app = application = bottle.default_app()
My Question:
Given the requirements, is this a reasonable approach? Or, do you recommend I use DB connection pooling? I am a little confused about using pooling since I am not sure at what level (nginx, uwsgi or bottle) does my app code get duplicated (to serve concurrent clients) and how should I go about creating the pool that I can use across different threads/processes (each of which contain a copy of this app code).
Obviously, this is my initial foray into web apps (and even serious Python for that matter) and would like to hear back from you if you think there is a better (I'm assuming there are many) ways to skin the cat.

python - sql query returns nothing

My json looks like this -
[
{
"Monitor Level":"1",
"Estimate SLA (+30)":"214",
"New Schedule":"",
"Job Name":"\\Job1\\jobgroup",
"Estimated Duration":"183",
"Actual Duration":"184"
},
{
"Monitor Level":"1",
"Estimate SLA (+30)":"179",
"New Schedule":"8:00 PM",
"Job Name":"\\Job2\\jobgroup",
"Estimated Duration":"1349",
"Actual Duration":"1349"
}
]
I run the following code -
for o in json_object:
# get jobid
db = pyodbc.connect('DRIVER={SQL Server};SERVER=dvidbsql01\dev2008;DATABASE=Admiral;UID=Tidal;PWD=tidal97')
cur = db.cursor()
cur.execute("""select jobmst_id from jobmst where jobmst_prntname + '\\' + jobmst_name = ?""", o['Job Name'])
r= cur.fetchall()
print r
And r returns the value I want.
If I use the code I want to however -
sql_jobid = """
select jobmst_id 'Job ID' from jobmst where jobmst_prntname + '\\' + jobmst_name = ?
"""
## DEFINE ENVIRONMENT DATABASES
def db():
if args.environment == 'DEV':
return pyodbc.connect('DRIVER={SQL Server};SERVER=server\instance;DATABASE=db;UID=user;PWD=pass')
## DEFINE UPDATE
def query_db(query, args=(), one=False):
cur = db().cursor()
cur.execute(query, args)
r = [dict((cur.description[i][0], value) \
for i, value in enumerate(row)) for row in cur.fetchall()]
cur.connection.close()
return (r[0] if r else None) if one else r
for o in json_object:
# get jobid
jobid = query_db(sql_jobid, (o['Job Name'][0]))
print jobid
It is not printing the value I want even though it's doing the same thing. even replacing o['Job Name'][0] with 'Job1\jobgroup' still doesn't return anything so it's something with my more pythonic code that seems to not want to parse the Job Name.

In the following line,
jobid = query_db(sql_jobid, (o['Job Name'][0]))
(o['Job Name'][0]) is not a tuple. If you want to pass a tuple, you need to append a trailing comma.
jobid = query_db(sql_jobid, (o['Job Name'][0],))
# ^