We Keep Coding

python - Not all parameters were used in the SQL statement

This function always gives me the following errors when run:
CRITICAL >> Not all parameters were used in the SQL statement
CRITICAL >> Exception for errors programming errors
I've spent hours looking at the code but cannot find the problem - What is wrong?
#************ TestCase Table Insertion *********************
def insertUpdateTestCase(prev_max_weeknumber):
log.Log('insertUpdateTestCase START', 'info')
insertUpdateTestCase_start_time = datetime.now()
testcases = """INSERT INTO prtm_testcase (testplan_identifier, testcase_name, testcase_identifier, testcase_longidentifier, testcase_uri, globalconfiguration_identifier, weeknumber, localconfiguration_identifier)
VALUES
(%s,%s,%s,%s,%s,%s,%s,%s)
ON CONFLICT (testplan_identifier, testcase_identifier, globalconfiguration_identifier, localconfiguration_identifier, weeknumber)
DO
UPDATE SET testcase_name = EXCLUDED.testcase_name,
testcase_longidentifier=EXCLUDED.testcase_longidentifier,
testcase_uri=EXCLUDED.testcase_uri,
weeknumber=EXCLUDED.weeknumber """
# Define some variables for executing Select Query based on limits
offset = 0
per_query = 10000
while True:
#execute query based on limits using projects
cursor_remets.execute("select tsdata.ts_moduleid, coalesce(tsdata_extended.ts_objecttext,'Unknown') as ts_objecttext, tsdata.ts_objectidentifier, SUBSTRING_INDEX(tsdata.ts_variant, ',', -1) as after_comma_value, tsdata.weeknumber,SUBSTRING_INDEX(tsdata.ts_variant, ',', -1) as project_id,SUBSTRING_INDEX(tsdata.ts_variant, ',', 1) as gc_id from tsdata left join tsdata_extended on tsdata_extended.ts_objectidentifier = tsdata.ts_objectidentifier and tsdata.ts_moduleid = tsdata_extended.ts_moduleid and tsdata.ts_variant = tsdata_extended.ts_variant where tsdata.weeknumber=%s OFFSET %s", (prev_max_weeknumber, per_query, offset))
rows = cursor_remets.fetchall()
if len(rows) > 0:
for row in rows:
#print(row)
testcond = (row[0] and row[1] and row[2] and row[4] and row[5] and row[6])
#testcond = True
if testcond:
cursor_prtm.execute(testcases,(row[0],row[1].replace("\x00", "\uFFFD").replace('\\', '\\\\'),row[2],None,None,row[6],row[4],row[5]))
conn_prtm.commit()
DataMigration.validateInsertedRecord('insertUpdateTestCase', row)
else:
log.Log('In insertUpdateTestCase, row validation failed ' + str(row), 'info')
else:
break
#print(str(len(rows)) + ' rows written successfully to prtm_testcase')
offset += per_query
log.Log('insertUpdateTestCase completed execution in ' + str(datetime.now()-insertUpdateTestCase_start_time), 'info')
Executed the SQL statement in the source database and rows are returned with no nulls or empty fields in the data.
Compared the list of fields against the defined statement and count the same number of parameters in both

PostgreSQL OLD not working in after update statement level trigger

I'm trying to update course points by the sum of course's lessons points.
It is working perfectly if I do select particular course ID like this:
BEGIN
UPDATE course
SET points = (SELECT COALESCE(SUM("lesson"."points"), 0) AS "sum_points" FROM "course" LEFT OUTER JOIN "lesson" ON ("course"."id" = "lesson"."course_id") WHERE "course"."id" = 7)
WHERE "course"."id" = 7;
RETURN NULL;
END;
But not working with OLD which is the updating instance. I want to update points of whichever course is being updated.
BEGIN
UPDATE course
SET points = (SELECT COALESCE(SUM("lesson"."points"), 0) AS "sum_points" FROM "course" LEFT OUTER JOIN "lesson" ON ("course"."id" = "lesson"."course_id") WHERE "course"."id" = OLD."course_id")
WHERE "course"."id" = OLD."course_id";
RETURN NULL;
END;
I'm using django-pgtriggers: https://pypi.org/project/django-pgtrigger/
#pgtrigger.register(
pgtrigger.Trigger(
name="add_course_point",
level=pgtrigger.Statement,
when=pgtrigger.After,
operation=pgtrigger.Update,
func=f"""
UPDATE course
SET points = (SELECT COALESCE(SUM("lesson"."points"), 0) AS "sum_points" FROM "course" LEFT OUTER JOIN "lesson" ON ("course"."id" = "lesson"."course_id") WHERE "course"."id" = OLD."course_id")
WHERE "course"."id" = OLD."course_id";
RETURN NULL;
"""
)
)

OLD and NEW are always NULL in case of Statement level.
Replace level=pgtrigger.Statement with level=pgtrigger.Row
#pgtrigger.register(
pgtrigger.Trigger(
name="add_course_point",
level=pgtrigger.Row,
when=pgtrigger.After,
operation=pgtrigger.Update,
func=f"""
UPDATE course
SET points = (SELECT COALESCE(SUM("lesson"."points"), 0) AS "sum_points" FROM "course" LEFT OUTER JOIN "lesson" ON ("course"."id" = "lesson"."course_id") WHERE "course"."id" = OLD."course_id")
WHERE "course"."id" = OLD."course_id";
RETURN NULL;
"""
)
)
or add referencing=pgtrigger.Referencing(old='old_table_name') and then modify your function.

I have registered 2 triggers. The first one is before update trigger for inserted lessons. The second is after update trigger for updated and deleted lessons. Course points are incremented or decremented by the sum of its lessons points after all.
#pgtrigger.register(
pgtrigger.Trigger(
name="add_course_point",
operation=pgtrigger.Insert,
level=pgtrigger.Row,
when=pgtrigger.Before,
func=f"""
UPDATE course
SET points = (points + NEW.points)
WHERE "course"."id" = NEW.course_id;
RETURN NEW;
""",
),
pgtrigger.Trigger(
name="update_course_point",
operation=(pgtrigger.Update | pgtrigger.Delete),
level=pgtrigger.Row,
when=pgtrigger.After,
func=f"""
UPDATE course
SET points = (SELECT COALESCE(SUM("lesson"."points"), 0) AS "sum_points" FROM "course" LEFT OUTER JOIN "lesson" ON ("course"."id" = "lesson"."course_id") WHERE "course"."id" = OLD.course_id)
WHERE "course"."id" = OLD.course_id;
RETURN NULL;
""",
),
)

How create a sqlalchemy delete query with multiples parameter from a loop

I'm new in python and sqlalchemy.
I already have a delete method working if I construct the where conditions by hand.
Now, I need to read the columns and values from an enter request in yaml format and create the where conditions.
#enter data as yaml
items:
- item:
table: [MyTable,OtherTable]
filters:
field_id: 1234
#other_id: null
Here is what I try and can't go ahead:
for i in use_case_cfg['items']:
item = i.get('item')
for t in item['table']:
if item['filters']:
filters = item['filters']
where_conditions = ''
count = 0
for column, value in filters.items():
aux = str(getattr(t, column) == bindparam(value))
if count == 0:
where_conditions += aux
else:
where_conditions += ', ' + aux
count += 1
to_delete = inv[t].__table__.delete().where(text(where_conditions))
#to_delete = t.__table__.delete().where(getattr(t, column) == value)
else:
to_delete = inv[t].__table__.delete()
CoreData.session.execute(to_delete)
To me, it looks ok, but when I run, I got the error below:
sqlalchemy.exc.StatementError: (sqlalchemy.exc.InvalidRequestError) A value is required for bind parameter '9876'
[SQL: DELETE FROM MyTable WHERE "MyTable".field_id = %(1234)s]
[parameters: [{}]]
(Background on this error at: http://sqlalche.me/e/cd3x)
Can someone explain to me what is wrong or the proper way to do it?
Thanks.

There are two problems with the code.
Firstly,
str(getattr(t, column) == bindparam(value))
is binding the value as a placeholder, so you end up with
WHERE f2 = :Bob
but it should be the name that maps to the value in filters (so the column name in your case), so you end up with
WHERE f2 = :f2
Secondly, multiple WHERE conditions are being joined with a comma, but you should use AND or OR, depending on what you are trying to do.
Given a model Foo:
class Foo(Base):
__tablename__ = 'foo'
id = sa.Column(sa.Integer, primary_key=True)
f1 = sa.Column(sa.Integer)
f2 = sa.Column(sa.String)
Here's a working version of a segment of your code:
filters = {'f1': 2, 'f2': 'Bob'}
t = Foo
where_conditions = ''
count = 0
for column in filters:
aux = str(getattr(t, column) == sa.bindparam(column))
if count == 0:
where_conditions += aux
else:
where_conditions += ' AND ' + aux
count += 1
to_delete = t.__table__.delete().where(sa.text(where_conditions))
print(to_delete)
session.execute(to_delete, filters)
If you aren't obliged to construct the WHERE conditions as strings, you can do it like this:
where_conditions = [(getattr(t, column) == sa.bindparam(column))
for column in filters]
to_delete = t.__table__.delete().where(sa.and_(*where_conditions))
session.execute(to_delete, filters)

slqalchemy core pagination with flask when we pass text fragments to generate the sql and python

For a simple select, pagination works as implemented here:
mheader_dict = dict(request.headers)
no_of_pgs = 0
if 'Maxpage' in mheader_dict.keys():
max_per_pg = int(mheader_dict['Maxpage'])
else:
max_per_pg = 100
page_no = int(request.headers.get('Pageno', type=int, default=1))
offset1 = (page_no - 1) * max_per_pg
s = select[orders]
if s is not None:
s = s.limit(max_per_pg).offset(offset1)
rs = g.conn.execute(s)
Conn is the connection object above
When text is used in the select statement, How to specify the limit?.How to rectify in below:
s1 = text('select d.*, (select array(select localities.name from localities, localities_boys where localities.id = localities_boys.locality_id and localities_boys.boy_id = d.id and localities_boys.boy_id is not null )) from delivery_boys d order by d.id;')
page_no = int(request.headers.get('Pageno', type=int, default=1))
offset1 = (page_no - 1) * max_per_pg
s1 = s1.limit(max_per_pg).offset(offset1)
rs1 = g.conn.execute(s1)
If s1 = s1.compile(engine) is used, it returns sqlalchemy.dialects.postgresql.psycopg2.PGCompiler_psycopg2 object which doesn't have limit functionality
How to convert sqlalchemy.sql.elements.TextClause to sqlalchemy.sql.selectable.Select using sqlalchemy core 1.0.8 to solve the above?
using sqlalchemy core v. 1.0.8, python 2.7,flask 0.12

Converted the TextClause to Select as :
s1 = select([text('d.*, (select array(select localities.name from localities, localities_boys where localities.id = localities_boys.locality_id and localities_boys.boy_id = d.id and localities_boys.boy_id is not null)) from delivery_boys d')])
Hence able to use limit and offset on the generated Select object

Using variables in Triple Quotes (Python)

I'm trying to input 2 variables into an sql query like so:
query = """
Select Distinct
sp.NAME,
sp.STUDY,
sp.DISEASE_ONTOLOGY_TERM,
sv.GENE,
sv.CDS_EFFECT,
sv.PROTEIN_EFFECT,
rep.STATUS,
sv.FRACTION_READS,
sv.DEPTH,
cvmship.REMOVED
From
SPECIMEN sp
inner join CURATION_MANAGER cm on (cm.SPECIMEN_ID = sp.SPECIMEN_ID)
inner join CURATION_VERSION cv on (cv.CURATION_VERSION_ID = cm.LATEST_VERSION_ID)
inner join CURATION_VERSION_MEMBERSHIP cvmship on (cvmship.VERSION_ID = cv.CURATION_VERSION_ID)
inner join CURATION_VERSION_MEMBER cvmer on (cvmer.CURATION_VERSION_MEMBER_ID = cvmship.MEMBER_ID)
inner join REPORTABLE rep on (rep.CURATION_VERSION_MEMBER_ID = cvmer.CURATION_VERSION_MEMBER_ID)
inner join SHORT_VARIANT sv on (sv.REPORTABLE_ID = rep.CURATION_VERSION_MEMBER_ID)
inner join (
Select
sp.SPECIMEN_ID,
cqr.STATUS
From
SPECIMEN sp
inner join CURATION_MANAGER cm on (cm.SPECIMEN_ID = sp.SPECIMEN_ID)
inner join CURATION_VERSION cv on (cv.CURATION_VERSION_ID = cm.LATEST_VERSION_ID)
inner join CURATION_VERSION_MEMBERSHIP cvmship on (cvmship.VERSION_ID = cv.CURATION_VERSION_ID)
inner join CURATION_VERSION_MEMBER cvmer on (cvmer.CURATION_VERSION_MEMBER_ID = cvmship.MEMBER_ID)
inner join CURATION_QC_RESULT cqr on (cqr.CURATION_VERSION_MEMBER_ID = cvmer.CURATION_VERSION_MEMBER_ID)
) cqr on (cqr.SPECIMEN_ID = sp.SPECIMEN_ID)
Where sp.ASSIGNED_INDEX is not null
AND sp.NAME like 'TRF%'
AND LENGTH(sp.NAME) = 12
AND cv.STATUS = 'final'
AND (cqr.STATUS = 'Pass' or cqr.STATUS = 'Qualified')
AND sp.STUDY like '%CLINICAL%'
AND sv.GENE = '%s'
AND sv.PROTEIN_EFFECT = '%s'
order by sp.name desc
""" % (gene, proEff)
When I run this script I get:
File "fetchDEVDB.py", line 57, in <module>
""" % (gene, proEff)
ValueError: unsupported format character ''' (0x27) at index 1481
I was thinking maybe it's trying to interpret the % as a format character, but I tried using %% to surround the %s and I get the same error. Any ideas?
Thanks

You need to '%'-quote the LIKE expression, not the format character:
...AND sp.NAME like 'TRF%%'
...
...AND sp.STUDY like '%%CLINICAL%%'

In this
AND sp.NAME like 'TRF%'
AND LENGTH(sp.NAME) = 12
AND cv.STATUS = 'final'
AND (cqr.STATUS = 'Pass' or cqr.STATUS = 'Qualified')
AND sp.STUDY like '%CLINICAL%'
AND sv.GENE = '%s'
AND sv.PROTEIN_EFFECT = '%s'
I see two invalid formats:
TRF%
%CLINICAL%