We Keep Coding

How to keep continuously write to Oracle table even if table is not accessible?

I am trying to insert multiple records into the one Oracle table continuously. For which I have written below python script.
import cx_Oracle
import config
connection = None
try:
# Make a connection
connection = cx_Oracle.connect(
config.username,
config.password,
config.dsn,
encoding=config.encoding)
# show the version of the Oracle Database
print(connection.version)
# Insert 20000 records
for i in range(1, 20001):
cursor = connection.cursor()
sql = "INSERT into SCHEMA.ABC (EVENT_ID, EVENT_TIME) VALUES( "+ str(i)+" , CURRENT_TIMESTAMP)"
cursor.execute(sql)
connection.commit()
except cx_Oracle.Error as error:
print(error)
finally:
if connection:
connection.close()
So, During the insert, when I change the table name it just create an exception and comes out of script(as the table is not available and cannot write). What I want is, Even if when I do the rename and table is not available, the script needs to keep continuously trying insert. Is there a way this can be possible?

Here's an example of what Ptit Xav was talking about. I added some code to quit after a max number of retries, since that's often desirable.
# Insert 20000 records
for i in range(1, 20001):
retry_count = 0
data_inserted = False
while not data_inserted:
try:
cursor = connection.cursor()
sql = "INSERT into SCHEMA.ABC (EVENT_ID, EVENT_TIME) VALUES( "+ str(i)+" , CURRENT_TIMESTAMP)"
cursor.execute(sql)
connection.commit()
data_inserted = True
except cx_Oracle.Error as error:
print(error)
time.sleep(5) # wait for 5 seconds between retries
retry_count += 1
if retry_count > 100:
print(f"Retry count exceeded on record {i}, quitting")
break
else:
# continue to next record if the data was inserted
continue
# retry count was exceeded; break the for loop.
break
See this answer for more explanation of the while... else logic.

You may want to encapsule the insert logik in a function that catches the possible exception and performs the retry
def safe_insert(con, i):
"""
insert a row with retry after exception
"""
retry_cnt = 0
sql_text = "insert into ABC(EVENT_ID, EVENT_TIME) VALUES(:EVENT_ID,CURRENT_TIMESTAMP) "
while True:
try:
with con.cursor() as cur:
cur.execute(sql_text, [i])
con.commit()
return
except cx_Oracle.Error as error:
print(f'error on inserting row {i}')
print(error)
time.sleep(1)
retry_cnt += 1
if (retry_cnt > 10):
raise error
Similar to #kfinity's answer I also added a limit on retry - if this limit is exceeded the function raise an exception.
Note also that the function uses bind variables in the INSERT statement which is preferable to the concatenation of the values in the statement.
The usage is as simple as
for i in range(1, 20001):
safe_insert(con, i)

Capture the Postgresql function return string within Python console

I have a postgresql function that returns a string as follows:
CREATE OR REPLACE FUNCTION script.fn_indent()
RETURNS character varying
LANGUAGE 'plpgsql'
------
----function body to perform data insertion job---
results:='0 - Success';
return results
exception when others then
get stacked diagnostics
v_state = returned_sqlstate,
v_msg = message_text,
v_detail = pg_exception_detail,
v_hint = pg_exception_hint,
v_context = pg_exception_context;
raise notice 'Transaction was rolled back';
raise notice '% %', SQLERRM, SQLSTATE;
results:=v_state||'-'||v_msg||v_msg||'-'||v_detail ||'-'||v_hint ||'-'||v_context;
return results;
Now I am trying to run the above function from python using psycopg2.
conn = psycopg2.connect({connection string})
curr = conn.cursor
try:
curr.execute("SELECT * FROM script.fn_indent())
conn.commit()
except (Exception, psycopg2.DatabaseError) as error:
err = str(error)
conn.rollback()
curr.close()
print(err)
conn.close()
The above code is running fine. But I want to capture the return string from script.fn_indent() and show the same to python console. Something like as below:
---above python script---
print (results) <--results is the returning string that comes from fn_indent()
How to do it? I do not have any clue on this.

I got the clue from this thread
Refer here
Accordingly, I have modified the code base as follows:
conn = psycopg2.connect({connection string})
curr = conn.cursor
curr.execute("SELECT * FROM script.fn_indent()")
conn.commit()
s = curr.fecthone()
print (s)
conn.close()

Insert Ordered Dict values while using MySQL INSERT INTO

I encounter this error when I'm trying to insert some values into a table.
Here's my code:
def tsx_insert(self, d_list):
for item in d_list:
query = """ INSERT IGNORE INTO tsx_first_insert(protocollo,procedura,oggetto,priorita,
tipo_richiesta,sottotipo_richiesta,emergenza,
richiesta,uo_richiedente,autore,scadenza_sla)
VALUES(%(protocollo)s,%(procedura)s,%(oggetto)s,%(priorita)s,%(tipo_richiesta)s,
%(sottotipo_richiesta)s,%(emergenza)s,%(richiesta)s,%(uo_richiedente)s,
%(autore)s,%(scadenza_sla)s)"""
values = item.values()
self.exec_query(query,values)
And here 'exec_query' function:
def exec_query(self, query, params):
try:
if self.connected is None:
self.connect()
self.cursor = self.connected.cursor()
self.cursor.connection.autocommit(True)
self.cursor.execute(query)
if self.cursor.description:
self.description = [d[0] for d in self.cursor.description]
self.rows = self.cursor.rowcount
self.sql_result = self.cursor.fetchall()
except MySQLdb.Error, e:
logging.error('Error {0}: {1}'.format(e.args[0], e.args[1]))
finally:
self.cursor.close()
The error is: "Error 1064: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '%(protocollo)s,%(procedura)s,%(oggetto)s,%(priorita)s,%(tipo_richiesta)s,
' at line 4"
I can't figure out what is the problem. Thank you in advance for your help.

You forgot to mention your params dictionary in your self.cursor.execute() method call, so the parameter strings were left in place rather than substituted.
Try
self.cursor.execute(query, params)

MySQL update does not update

I have a problem that the update mysql part is sometimes not updating a value and it appears to be random I can't find the cause for it. it takes MYSQL entries with key value "TRANSFER" and schould set the same key value to "EXECUTED". Sometimes iI have 20 processes that work fine, sometimes I have 10 processes and half didnt get updated.
try:
db = MySQLdb.connect(
host='localhost',
user='root',
passwd='pw',
db='db',
)
db.autocommit(True)
except Exception as e:
sys.exit("Can't connect to db")
cur = db.cursor()
setstatus = "EXECUTED"
for fn in os.listdir('.'):
if os.path.isfile(fn):
UUID = fn.replace(".ac", "")
try:
cur.execute("""
UPDATE olorequest
SET status = %s
WHERE UUID = %s
""", (setstatus, UUID))
except Exception as e:
raise IOError(e)
ftp.storlines('STOR ' + fn, open(fn, 'r+'))
try:
shutil.move(fn, executed_ac_files)
except Exception as e:
shutil.move(fn, error_files)
raise IOError(e)
time.sleep(5)

Basically the reason why a row is not updated in an UPDATE request is that the predicate of the WHERE clause is not met. Additionnaly, as you perform this action through a program, also check its logic and its reliability.

When to use DRCP and sessionpool in cx_oracle python?

For a specific use case - in which I have 100 databases and 1 database is the central database, now my app connects to that one central database which spawns connections to any of the 100 databases as per the request of the user to run some query on any of them.
In this case does using DRCP makes same as I dont want the connection to be killed if the user is running the query at the same time I dont want too many connections to be opened to the db which I control by creating a profile on the database which limits the number of active sessions to some low number say 5 for that specific user(read_only_user) using that specific profile(read_only_profile).
Right now I am using the standard open a connection per request model. But Im not sure if thats the best way to go about it.
import cx_Oracle
import logging, time
class Database(object):
'''
Use this method to for DML SQLS :
Inputs - Sql to be executed. Data related to that sql
Returns - The last inserted, updated, deleted ID.
'''
def __init__(self, user, password, host, port, service_name, mode, *args):
#mode should be 0 if not cx_Oracle.SYSDBA
self.user = user
self.password = password
self.host = host
self.port = port
self.user = user
self.service_name = service_name
self.logger = logging.getLogger(__name__)
try:
self.mode = mode
except:
self.mode = 0
self.logger.info(" Mode is not mentioned while creating database object")
self.connection = None
dsn = cx_Oracle.makedsn(self.host, self.port, self.service_name)
self.connect_string = self.user + '/' + self.password + '#' + dsn
try:
self.connection = cx_Oracle.connect(self.connect_string, mode=self.mode,
threaded=True)
self.connection.stmtcachesize = 1000
self.connection.client_identifier = 'my_app'
self.cursor = self.connection.cursor()
self.idVar = self.cursor.var(cx_Oracle.NUMBER)
except cx_Oracle.DatabaseError, exc:
error, = exc
self.logger.exception(
'Exception occured while trying to create database object : %s',
error.message)
raise exc
def query(self, q):
try:
self.cursor.execute(q)
return self.cursor.fetchall()
except cx_Oracle.DatabaseError, exc:
error, = exc
self.logger.info(
"Error occured while trying to run query: %s, error : %s", q,
error.message)
return error.message
def dml_query(self, sql):
try:
self.cursor.execute(sql)
self.connection.commit()
return 1
except Exception as e:
self.logger.exception(e)
return 0
def dml_query_with_data(self, sql, data):
"""
Use this method to for DML SQLS :
Inputs - Sql to be executed. Data related to that sql
Returns - The last inserted, updated, deleted ID.
"""
try:
self.cursor.execute(sql, data)
self.connection.commit()
return 1
except Exception as e:
self.logger.exception(e)
return 0
def update_output(self, clob, job_id, flag):
try:
q = "Select output from my_table where job_id=%d" % job_id
self.cursor.execute(q)
output = self.cursor.fetchall()
#Checking if we already have some output in the clob for that job_id
if output[0][0] is None:
if flag == 1:
self.cursor.execute("""UPDATE my_table
SET OUTPUT = :p_clob
,job_status=:status WHERE job_id = :p_key""",
p_clob=clob, status="COMPLETED", p_key=job_id)
else:
self.cursor.execute("""UPDATE my_table
SET OUTPUT = :p_clob
,job_status=:status WHERE job_id = :p_key""",
p_clob=clob, status="FAILED", p_key=job_id)
else:
self.cursor.execute("""UPDATE my_table
SET OUTPUT = OUTPUT || ',' || :p_clob
WHERE job_id = :p_key""", p_clob=clob, p_key=job_id)
self.connection.commit()
rows_updated = self.cursor.rowcount
return rows_updated
except Exception as e:
self.logger.exception(e)
return 0
def __del__(self):
try:
if self.connection is not None:
self.connection.close()
except Exception as e:
self.logger.exception(
"Exception while trying to close database connection object : %s", e)
'''
if __name__ == '__main__':
db = Database('test', 'test', 'my_host', '1000', 'my_db', 0)
columns = db.query('select * from my-table')
print columns
'''
This is my database class, and I create an object whenever I need a connect to the DB. And the init and del method take care of constructing and destructing the object.
Should I be using DRCP/ sessionPool to improve performance.
What if there are too many users waiting coz all the connections in DRCP are taken?
Can I have sessionPool per database (for the 100 databases, each database can take atmost 5 connections at a time for that read_only_user)