We Keep Coding

Python sqlite operation,SQL statement 'where field in (1,2)' syntax error

Python sqlite operation,SQL statement 'where field in (1,2)' syntax error
The error is:sqlite3.OperationalError: near ":id": syntax error
My search of the Official Python documentation and Google failed to find the answer:
https://docs.python.org/3/library/sqlite3.html
How should arguments be passed?
'''first create test.db:table and field
CREATE TABLE test_tab (
id INTEGER PRIMARY KEY ASC,
test_num INT,
test_field TEXT
);
'''
import sqlite3
con = sqlite3.connect('test.db')
con.set_trace_callback(print) # start the debug
d = [
(111,'aaa'),
(111,'bbb'),
(111,'ccc'),
(444,'ddd')
]
sql = "insert into `test_tab` (`test_num`, `test_field`) values (?,?)"
cursor = con.executemany(sql, d)
con.commit() # Execute successfully
#####################
# wrong code begin,why sql 'in ()' is wrong?
sql = "SELECT * from `test_tab` where `test_num`=:num AND `id` in :id"
par = {'num': 111, 'id': (1,2)} # The number of 'id' parameters is uncertain
x = con.execute(sql, par)
print(x.fetchall())

In the second query, you would need actually separate placeholders for every value in the IN clause. In addition, I would use ? here:
num = 111
ids = (1, 2)
par = (num,) + ids
sql = "select * from test_tab where test_num = ? AND id in "
in_clause = '(?' + ', ?'*(len(ids) - 1) + ')'
sql = sql + in_clause
x = con.execute(sql, par)
print(x.fetchall())
The SQL query generated by the above script is:
select * from test_tab where test_num = ? AND in (?, ?)
and we bind (111, 1, 2) to the three ? placeholders.

how to see full query with values in ponyorm

Today i am using the sql_debug(True) that helps me to see the queries but without the values.
How could i see how ponyorm translate the query with values ?
Thank you very much.
This is an example of query i'm using.
with db_session:
access = select(p for p in Access if raw_sql('( lower(first_name) = lower($first_name) and lower(last_name) = lower($last_name) ) '
'or ( lower(first_name) = lower($last_name) and lower(last_name) = lower($first_name) ) '
'or (lower(facebook_url) = lower($facebook_url)) '
'or (lower(twitter_url) = lower($twitter_url)) '
'or (lower(linkedin_url) = lower($linkedin_url)) '))
.order_by(desc(Access.twitter_url),desc(Access.facebook_url),desc(Access.linkedin_url),
desc(Access.facebook_url))
print(access.get_sql())

I use
logging.getLogger(__name__).debug('SQL:\n\n\t\t\t%s\n', '\n'.join(unicode(x) for x in request._construct_sql_and_arguments()[:2]).replace('\n', '\n\t\t\t'))
for that.
For example,
19:30:01.902 data.py:231 [DEBUG] SQL:
SELECT "x"."_id", "x"."filename", "x"."_created", "x"."_updated"
FROM "reports" "x"
WHERE "x"."_id" <= ?
AND "x"."_created" >= ?
(50, '2019-04-17 19:30:01.900028')
will be printed out.

You can use set_sql_debug(debug=True, show_values=True).
Reference here.

There is a method called get_sql()
query_obj = select(c for c in Category if c.name.startswith('v'))
sql = query_obj.get_sql()
print(sql)
output:
SELECT "c"."id", "c"."name"
FROM "category" "c"
WHERE "c"."name" LIKE 'v%%'
code continue:
for obj in query_obj:
print('id:', obj.id, 'name:', obj.name)
output:
id: 1 name: viki
here is a link to the docs https://docs.ponyorm.com/api_reference.html#Query.get_sql
You can log the sql or simply print it.
Update:
OP updated the question:
If the sql query has a variable like $name it is passed as a sql parameter.
first_name = 'viki'
query = select(c for c in Category if raw_sql('( lower(name) = lower($first_name))'))
query.get_sql()
so get_sql() will return the value with a placeholder, and the output will look like this:
'SELECT "c"."id", "c"."name", "c"."age"\nFROM "Category" "c"\nWHERE ( lower(name) = lower(?))'
If we want no placeholders should be there in the query then we can avoid passing direct sql to query and instead build it separately in python.
Like this:
query = select(c for C in Category if c.name == 'viki')
query.get_sql()
output:
'SELECT "c"."id", "c"."name", "c"."age"\nFROM "Category" "c"\nWHERE "c"."name" = \'viki\''

Trying to Convert SQL query to SQLAlchemy query

I'm trying to translate raw SQL into an sqlalchemy query, but so far I'm getting different results. The results I get from my attempt at sqlalchemy is missing Property objects (I get back a tuple with (ListingCalendarDays, None, PricingData)), so I think something is missing in the translation.
Original query:
result = session.execute("""SELECT p.id as property_id,
p.home_code,
c.listing_id,
c.calendar_date,
c.available,
ab.price
FROM listing_calendar_days c
LEFT JOIN properties p
ON (p.id::integer = c.listing_id and p.id is not null and p.id <> '')
LEFT JOIN pricing_data ab
ON c.listing_id = ab.listing_id
AND c.calendar_date = ab.price_date
WHERE c.calendar_date >= '%s'
AND c.calendar_date <= '%s' ;""" % ( now.isoformat(), end.isoformat()))
My Attempt:
query = (session.query( ListingCalendarDays, Properties, PricingData )
.outerjoin(Properties,
and_(Properties.id == str(ListingCalendarDays.listing_id),
Properties.id != None))
.outerjoin(PricingData,
and_(ListingCalendarDays.listing_id == PricingData.listing_id,
ListingCalendarDays.calendar_date == PricingData.price_date))
.filter(ListingCalendarDays.calendar_date.between( now, end )))
result = query.all()

You can print out the query that SQLAlchemy generates by doing
print(query)
In this particular case, your join condition for Properties is wrong. str(ListingCalendarDays.listing_id) produces the literal string "ListingCalendarDays.listing_id", not the column listing_id as casted to varchar. For a cast, you need the cast() construct:
cast(ListingCalendarDays.listing_id, String)
In addition, you're also missing the p.id <> '' condition.
Putting it all together:
and_(Properties.id == cast(ListingCalendarDays.listing_id, String),
Properties.id != None,
Properties.id != "")

Using % wildcard with pg8000

I have a query similar to below:
def connection():
pcon = pg8000.connect(host='host', port=1234, user='user', password='password', database = 'database')
return pcon, pcon.cursor()
pcon, pcur = connection()
query = """ SELECT * FROM db WHERE (db.foo LIKE 'string-%' OR db.foo LIKE 'bar-%')"""
db = pd.read_sql_query(query, pcon)
However when I try to run the code I get:
DatabaseError: '%'' not supported in a quoted string within the query string
I have tried escaping the symbol with \ and an additional % with no luck. How can I get pg8000 to treat this as a wildcard properly?

"In Python, % usually refers to a variable that follows the string. If you want a literal percent sign, then you need to double it. %%"
-- Source
LIKE 'string-%%'
Otherwise, if that doesn't work, PostgreSQL also supports underscores for pattern matching.
'abc' LIKE 'abc' true
'abc' LIKE 'a%' true
'abc' LIKE '_b_' true
But, as mentioned in the comments,
An underscore (_) in pattern stands for (matches) any single character; a percent sign (%) matches any sequence of zero or more characters
According to the source code, though, it would appear the problem is the single quote following the % in your LIKE statement.
if next_c == "%":
in_param_escape = True
else:
raise InterfaceError(
"'%" + next_c + "' not supported in a quoted "
"string within the query string")
So if next_c == "'" instead of next_c == "%", then you would get your error
'%'' not supported in a quoted string within the query string

With a recent version of pg8000 you shouldn't have any problems with a % in a LIKE. For example:
>>> import pg8000.dbapi
>>>
>>> con = pg8000.dbapi.connect(user="postgres", password="cpsnow")
>>> cur = con.cursor()
>>> cur.execute("CREATE TEMPORARY TABLE book (id SERIAL, title TEXT)")
>>> for title in ("Ender's Game", "The Magus"):
... cur.execute("INSERT INTO book (title) VALUES (%s)", [title])
>>>
>>> cur.execute("SELECT * from book WHERE title LIKE 'The %'")
>>> cur.fetchall()
([2, 'The Magus'],)

Python MySQL escape special characters

I am using python to insert a string into MySQL with special characters.
The string to insert looks like so:
macaddress_eth0;00:1E:68:C6:09:A0;macaddress_eth1;00:1E:68:C6:09:A1
Here is the SQL:
UPGRADE inventory_server
set server_mac = macaddress\_eth0\;00\:1E\:68\:C6\:09\:A0\;macaddress\_eth1\;00\:1E\:68\:C6\:09\:A1'
where server_name = 'myhost.fqdn.com
When I execute the update, I get this error:
ERROR 1064 (42000):
You have an error in your SQL syntax; check the manual that corresponds to your
MySQL server version for the right syntax to use near 'UPGRADE inventory_server
set server_mac = 'macaddress\_eth0\;00\:1E\:68\:C6\:09\' at line 1
The python code:
sql = 'UPGRADE inventory_server set server_mac = \'%s\' where server_name = \'%s\'' % (str(mydb.escape_string(macs)),host)
print sql
try:
con = mydb.connect(DBHOST,DBUSER,DBPASS,DB);
with con:
cur = con.cursor(mydb.cursors.DictCursor)
cur.execute(sql)
con.commit()
except:
return False
How can I insert this text raw?

This is one of the reasons you're supposed to use parameter binding instead of formatting the parameters in Python.
Just do this:
sql = 'UPGRADE inventory_server set server_mac = %s where server_name = %s'
Then:
cur.execute(sql, macs, host)
That way, you can just deal with the string as a string, and let the MySQL library figure out how to quote and escape it for you.
On top of that, you generally get better performance (because MySQL can compile and cache one query and reuse it for different parameter values) and avoid SQL injection attacks (one of the most common ways to get yourself hacked).

Welcome to the world of string encoding formats!
tl;dr - The preferred method for handling quotes and escape characters when storing data in MySQL columns is to use parameterized queries and let the MySQLDatabase driver handle it. Alternatively, you can escape quotes and slashes by doubling them up prior to insertion.
Full example at bottom of link
standard SQL update
# as_json must have escape slashes and quotes doubled
query = """\
UPDATE json_sandbox
SET data = '{}'
WHERE id = 1;
""".format(as_json)
with DBConn(*client.conn_args) as c:
c.cursor.execute(query)
c.connection.commit()
parameterized SQL update
# SQL Driver will do the escaping for you
query = """\
UPDATE json_sandbox
SET data = %s
WHERE id = %s;
"""
with DBConn(*client.conn_args) as c:
c.cursor.execute(query, (as_json, 1))
c.connection.commit()
Invalid JSON SQL
{
"abc": 123,
"quotes": "ain't it great",
"multiLine1": "hello\nworld",
"multiLine3": "hello\r\nuniverse\r\n"
}
Valid JSON SQL
{
"abc": 123,
"quotes": "ain''t it great",
"multiLine1": "hello\\nworld",
"multiLine3": "hello\\r\\nuniverse\\r\\n"
}
Python transform:
# must escape the escape characters, so each slash is doubled
# Some MySQL Python libraries also have an escape() or escape_string() method.
as_json = json.dumps(payload) \
.replace("'", "''") \
.replace('\\', '\\\\')
Full example
import json
import yaml
from DataAccessLayer.mysql_va import get_sql_client, DBConn
client = get_sql_client()
def encode_and_store(payload):
as_json = json.dumps(payload) \
.replace("'", "''") \
.replace('\\', '\\\\')
query = """\
UPDATE json_sandbox
SET data = '{}'
WHERE id = 1;
""".format(as_json)
with DBConn(*client.conn_args) as c:
c.cursor.execute(query)
c.connection.commit()
return
def encode_and_store_2(payload):
as_json = json.dumps(payload)
query = """\
UPDATE json_sandbox
SET data = %s
WHERE id = %s;
"""
with DBConn(*client.conn_args) as c:
c.cursor.execute(query, (as_json, 1))
c.connection.commit()
return
def retrieve_and_decode():
query = """
SELECT * FROM json_sandbox
WHERE id = 1
"""
with DBConn(*client.conn_args) as cnx:
cursor = cnx.dict_cursor
cursor.execute(query)
rows = cursor.fetchall()
as_json = rows[0].get('data')
payload = yaml.safe_load(as_json)
return payload
if __name__ == '__main__':
payload = {
"abc": 123,
"quotes": "ain't it great",
"multiLine1": "hello\nworld",
"multiLine2": """
hello
world
""",
"multiLine3": "hello\r\nuniverse\r\n"
}
encode_and_store(payload)
output_a = retrieve_and_decode()
encode_and_store_2(payload)
output_b = retrieve_and_decode()
print("original: {}".format(payload))
print("method_a: {}".format(output_a))
print("method_b: {}".format(output_b))
print('')
print(output_a['multiLine1'])
print('')
print(output_b['multiLine2'])
print('\nAll Equal?: {}'.format(payload == output_a == output_b))

Python example how to insert raw text:
Create a table in MySQL:
create table penguins(id int primary key auto_increment, msg VARCHAR(4000))
Python code:
#!/usr/bin/env python
import sqlalchemy
from sqlalchemy import text
engine = sqlalchemy.create_engine(
"mysql+mysqlconnector://yourusername:yourpassword#yourhostname.com/your_database")
db = engine.connect()
weird_string = "~!##$%^&*()_+`1234567890-={}|[]\;':\""
sql = text('INSERT INTO penguins (msg) VALUES (:msg)')
insert = db.execute(sql, msg=weird_string)
db.close()
Run it, examine output:
select * from penguins
1 ~!##$%^&*()_+`1234567890-={}|[]\;\':"
None of those characters were interpreted on insert.

Although I also think parameter binding should be used, there is also this:
>>> import MySQLdb
>>> example = r"""I don't like "special" chars ¯\_(ツ)_/¯"""
>>> example
'I don\'t like "special" chars \xc2\xaf\\_(\xe3\x83\x84)_/\xc2\xaf'
>>> MySQLdb.escape_string(example)
'I don\\\'t like \\"special\\" chars \xc2\xaf\\\\_(\xe3\x83\x84)_/\xc2\xaf'