We Keep Coding

How can I (safely) convert a dictionary into an UPDATE statement for sqlite in Python?

Say for example, I have a table of students, and I have a Python dictionary
mydict = {"fname" : "samwise", "lname" : "gamgee", "age" : 13}
How can I safely generate a Python function that can UPDATE this into my student table? (In my use-case I can safely assume that the student already exists in the table, AND I KNOW the id already)
I have created a function that achieves this functionality, but I can't help but think it's a bit crude, and perhaps open to SQL injection attacks
def sqlite_update(conn, table, data, pkeyname, pkeyval):
set_lines = ""
for k,v in data.items():
set_lines += "{} = '{}', ".format(k,v)
set_lines = set_lines[:-2] #remove space and comma from last item
sql = "UPDATE {0} SET {1} WHERE {2} = '{3}'"
statement = sql.format(table, set_lines, pkeyname, pkeyval)
conn.execute(statement)
conn.commit()
And to update I just call
sqlite_update(conn, "student", mydict, "id", 1)

As I assume you are using sqlalchemy. In this case, you can use sqlalchemy.sql.text function which escapes strings if required.
You can try to adjust your function as below.
from sqlalchemy.sql import text
def sqlite_update(conn, table, data, pkeyname, pkeyval):
set_lines = ",".join([f"{k}=:{k}" for k in data.keys()])
statement = text(f"UPDATE {table} SET {set_lines} WHERE {pkeyname} = :pkeyval")
args = dict(data)
args["pkeyval"] = pkeyval
conn.execute(statement, args)
conn.commit()
For more details, refer to sqlalchemy official documentation on text function.
EDIT
As for sqlite3 connection you can do basically the same thing as above, with slight changes.
def sqlite_update(conn, table, data, pkeyname, pkeyval):
set_lines = ",".join([f"{k}=:{k}" for k in data.keys()])
statement = f"UPDATE {table} SET {set_lines} WHERE {pkeyname} = :pkeyval"
args = dict(data)
args["pkeyval"] = pkeyval
conn.execute(statement, args)
conn.commit()
Refer to sqlite3 execute

This is indeed widely opened to SQL injection, because you build the query as a string including its values, instead of using a parameterized query.
Building a parameterized query with Python is easy:
def sqlite_update(conn, table, data, pkeyname, pkeyval):
query = f"UPDATE {table} SET " + ', '.join(
"{}=?".format(k) for k in data.keys()) + f" WHERE {pkeyname}=?"
# uncomment next line for debugging
# print(query, list(data.values()) + [pkeyval])
conn.execute(query, list(data.values()) + [pkeyval])
With your example, the query displayed by the debug print line is:
UPDATE student SET fname=?, lname=?, age=? WHERE id=?
with the following values list: ['samwise', 'gamgee', 13, 1]
But beware, to be fully protected from SQL injection, you should sanitize the table and field names to ensure they contain no dangerous characters like ;

Using psycopg2 to insert list of tuples which includes a JSON array field into Postgres database

I'm trying to insert rows into a database that has three columns: publication, publication_year, and authors. The field authors is of type JSON []. Currently, I have the following code:
query = "INSERT INTO " + store_table_name + "( " + ",".join(tuple(key_types.keys())) + ") VALUES %s"
try:
# connect to the database
conn, cur = pg_connect(DB_OSP, db_host, db_user, db_password)
conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
logger.info('Adding %s rows to %s' % (len(data), store_table_name))
execute_values(cur, query, data)
cur.close()
conn.commit()
# exception handling to follow
Data is a list of tuples where each entry is (publication, publication_year, authors). So for example, an entry could look like:
('Sample Book', 2019, ['{"Author1_fname": "Billy", "Author1_lname": "Bob"}', '{"Author2_fname": "King", "Author2_lname": "Kong"}'])
Initially, the code doesn't work as there is a type mismatch (it tries to insert authors as a text array instead of a JSON array). I tried changing the array to string type which didn't work either. I'm not quite sure how to explicitly add the ":::json[]" cast using execute_values. Does anyone have experience doing this?

Try using Json from psycopg2.extras by formatting your data as:
(
"Sample Book",
2019,
Json([{"Author1_fname": "Billy", "Author1_lname": "Bob"}, {"Author2_fname": "King", "Author2_lname": "Kong"}])
)

insert into mysql database with pymysql failing to insert

I'm trying to insert dummy data into a mysql database.
The database structure looks like:
database name: messaround
database table name: test
table structure:
id (Primary key, auto increment)
path (varchar(254))
UPDATED 2 method below, and error.
I have a method to try to insert via:
def insert_into_db(dbcursor, table, *cols, **vals):
try:
query = "INSERT INTO {} ({}) VALUES ('{}')".format(table, ",".join(cols), "'),('".join(vals))
print(query)
dbcursor.execute(query)
dbcursor.commit()
print("inserted!")
except pymysql.Error as exc:
print("error inserting...\n {}".format(exc))
connection=conn_db()
insertstmt=insert_into_db(connection, table='test', cols=['path'], vals=['test.com/test2'])
However, this is failing saying:
INSERT INTO test () VALUES ('vals'),('cols')
error inserting...
(1136, "Column count doesn't match value count at row 1")
Can you please assist?
Thank you.

If you use your code:
def insert_into_db(dbcursor, table, *cols, **vals):
query = "INSERT INTO {} ({}) VALUES ({})".format(table,",".join(cols), ",".join(vals))
print(query)
insert_into_db('cursor_here', 'table_here', 'name', 'city', name_person='diego', city_person='Sao Paulo')
Python returns:
INSERT INTO table_here (name,city) VALUES (name_person,city_person)
Now with this other:
def new_insert_into_db(dbcursor, table, *cols, **vals):
vals2 = ''
for first_part, second_part in vals.items():
vals2 += '\'' + second_part + '\','
vals2 = vals2[:-1]
query = "INSERT INTO {} ({}) VALUES ({})".format(table,",".join(cols), vals2)
print(query)
new_insert_into_db('cursor_here', 'table_here', 'name', 'city', name_person='diego', city_person='Sao Paulo')
Python will return the correct SQL:
INSERT INTO table_here (name,city) VALUES ('diego','Sao Paulo')

Generally in Python you pass a parameterized query to the DB driver. See this example in PyMySQL's documentation; it constructs the INSERT query with placeholder characters, then calls cursor.execute() passing the query, and a tuple of the actual values.
Using parameterized queries is also recommended for security purposes, as it defeats many common SQL injection attacks.

you should print the sql statement which you've generated, that makes it a lot easier to see what's wrong.
But I guess you need quotes ' around string values for your ",".join(vals) (in case there are string values.
So your code is producing
insert into test (path,) values (test.com/test2,);
but it should produce
insert into test (`path`) values ('test.com/test2');
Otherwise try https://github.com/markuman/MariaSQL/ which makes it super easy to insert data to MariaDB/MySQL using pymysql.

Change your query as below
query = "INSERT INTO {} ({}) VALUES ('{}')".format(table, ",".join(cols), "'),('".join(vals))
As you are using join, the variable is expected to be a list but not a string
table = 'test'
cols = ['path']
vals = ['test.com/test2', 'another.com/anothertest']
print(query)
"INSERT INTO test (path) VALUES ('test.com/test2'),('another.com/anothertest')"
Update:
def insert_into_db(dbconnection=None, table='', cols=None, vals=None):
mycursor = dbconnection.cursor()
if not (dbconnection and table and cols and vals):
print('Must need all values')
quit()
try:
query = "INSERT INTO {} ({}) VALUES ('{}')".format(table, ",".join(cols), "'),('".join(vals))
mycursor.execute(query)
dbconnection.commit()
print("inserted!")
except pymysql.Error as exc:
print("error inserting...\n {}".format(exc))
connection=conn_db()
insertstmt=insert_into_db(dbconnection=connection, table='test', cols=['path'], vals=['test.com/test2'])

Why does Psycopg2 return list of tuples in with Stored Procedure?

I have been using Psycopg2 to read stored procedures from Postgres successfully and getting a nice tuple returned, which has been easy to deal with. For example...
def authenticate(user, password):
conn = psycopg2.connect("dbname=MyDB host=localhost port=5433 user=postgres password=mypwd")
cur = conn.cursor()
retrieved_pwd = None
retrieved_userid = None
retrieved_user = None
retrieved_teamname = None
cur.execute("""
select "email", "password", "userid", "teamname"
from "RegisteredUsers"
where "email" = '%s'
""" % user)
for row in cur:
print row
The row that prints would give me ('user#gmail.com ', '84894531656894hashedpassword5161651165 ', 36, 'test ')
However, when I run the following code to read a row of fixtures with a Stored Procedure, I get (what looks to me like) an unholy mess.
def get_from_sql(userid):
conn = psycopg2.connect("dbname=MyDB host=localhost port=5433 user=postgres password=pwd")
fixture_cursor = conn.cursor()
callproc_params = [userid]
fixture_cursor.execute("select sppresentedfixtures(%s)", callproc_params)
for row in fixture_cursor:
print row
The resulting output:
('(5,"2015-08-28 21:00:00","2015-08-20 08:00:00","2015-08-25 17:00:00","Team ",,"Team ",,"Final ")',)
I have researched the cursor class and cannot understand why it outputs like this for a stored procedure. When executing within Postgres, the output is in a perfect Tuple. Using Psycopg2 adds onto the tuple and I don't understand why?
How do I change this so I get a tidy tuple? What am I not understanding about the request that I am making that gives me this result?
I have tried the callproc function and get an equally unhelpful output. Any thoughts on this would be great.

This is because you're SELECTing the result of the function directly. Your function returns a set of things, and each "thing" happens to be a tuple, so you're getting a list of stringified tuples back. What you want is this:
SELECT * FROM sppresentedfixtures(...)
But this doesn't work, because you'll get the error:
ERROR: a column definition list is required for functions returning "record"
The solution is to return a table instead:
CREATE OR REPLACE FUNCTION sppresentedfixtures(useridentity integer) RETURNS TABLE(
Fixture_No int,
Fixture_Date timestamp,
...
) AS
$BODY$
select
"Fixtures"."Fixture_No",
"Fixtures"."Fixture_Date",
...
from "Fixtures" ...
$BODY$ LANGUAGE sql

Python MySQL escape special characters

I am using python to insert a string into MySQL with special characters.
The string to insert looks like so:
macaddress_eth0;00:1E:68:C6:09:A0;macaddress_eth1;00:1E:68:C6:09:A1
Here is the SQL:
UPGRADE inventory_server
set server_mac = macaddress\_eth0\;00\:1E\:68\:C6\:09\:A0\;macaddress\_eth1\;00\:1E\:68\:C6\:09\:A1'
where server_name = 'myhost.fqdn.com
When I execute the update, I get this error:
ERROR 1064 (42000):
You have an error in your SQL syntax; check the manual that corresponds to your
MySQL server version for the right syntax to use near 'UPGRADE inventory_server
set server_mac = 'macaddress\_eth0\;00\:1E\:68\:C6\:09\' at line 1
The python code:
sql = 'UPGRADE inventory_server set server_mac = \'%s\' where server_name = \'%s\'' % (str(mydb.escape_string(macs)),host)
print sql
try:
con = mydb.connect(DBHOST,DBUSER,DBPASS,DB);
with con:
cur = con.cursor(mydb.cursors.DictCursor)
cur.execute(sql)
con.commit()
except:
return False
How can I insert this text raw?

This is one of the reasons you're supposed to use parameter binding instead of formatting the parameters in Python.
Just do this:
sql = 'UPGRADE inventory_server set server_mac = %s where server_name = %s'
Then:
cur.execute(sql, macs, host)
That way, you can just deal with the string as a string, and let the MySQL library figure out how to quote and escape it for you.
On top of that, you generally get better performance (because MySQL can compile and cache one query and reuse it for different parameter values) and avoid SQL injection attacks (one of the most common ways to get yourself hacked).

Welcome to the world of string encoding formats!
tl;dr - The preferred method for handling quotes and escape characters when storing data in MySQL columns is to use parameterized queries and let the MySQLDatabase driver handle it. Alternatively, you can escape quotes and slashes by doubling them up prior to insertion.
Full example at bottom of link
standard SQL update
# as_json must have escape slashes and quotes doubled
query = """\
UPDATE json_sandbox
SET data = '{}'
WHERE id = 1;
""".format(as_json)
with DBConn(*client.conn_args) as c:
c.cursor.execute(query)
c.connection.commit()
parameterized SQL update
# SQL Driver will do the escaping for you
query = """\
UPDATE json_sandbox
SET data = %s
WHERE id = %s;
"""
with DBConn(*client.conn_args) as c:
c.cursor.execute(query, (as_json, 1))
c.connection.commit()
Invalid JSON SQL
{
"abc": 123,
"quotes": "ain't it great",
"multiLine1": "hello\nworld",
"multiLine3": "hello\r\nuniverse\r\n"
}
Valid JSON SQL
{
"abc": 123,
"quotes": "ain''t it great",
"multiLine1": "hello\\nworld",
"multiLine3": "hello\\r\\nuniverse\\r\\n"
}
Python transform:
# must escape the escape characters, so each slash is doubled
# Some MySQL Python libraries also have an escape() or escape_string() method.
as_json = json.dumps(payload) \
.replace("'", "''") \
.replace('\\', '\\\\')
Full example
import json
import yaml
from DataAccessLayer.mysql_va import get_sql_client, DBConn
client = get_sql_client()
def encode_and_store(payload):
as_json = json.dumps(payload) \
.replace("'", "''") \
.replace('\\', '\\\\')
query = """\
UPDATE json_sandbox
SET data = '{}'
WHERE id = 1;
""".format(as_json)
with DBConn(*client.conn_args) as c:
c.cursor.execute(query)
c.connection.commit()
return
def encode_and_store_2(payload):
as_json = json.dumps(payload)
query = """\
UPDATE json_sandbox
SET data = %s
WHERE id = %s;
"""
with DBConn(*client.conn_args) as c:
c.cursor.execute(query, (as_json, 1))
c.connection.commit()
return
def retrieve_and_decode():
query = """
SELECT * FROM json_sandbox
WHERE id = 1
"""
with DBConn(*client.conn_args) as cnx:
cursor = cnx.dict_cursor
cursor.execute(query)
rows = cursor.fetchall()
as_json = rows[0].get('data')
payload = yaml.safe_load(as_json)
return payload
if __name__ == '__main__':
payload = {
"abc": 123,
"quotes": "ain't it great",
"multiLine1": "hello\nworld",
"multiLine2": """
hello
world
""",
"multiLine3": "hello\r\nuniverse\r\n"
}
encode_and_store(payload)
output_a = retrieve_and_decode()
encode_and_store_2(payload)
output_b = retrieve_and_decode()
print("original: {}".format(payload))
print("method_a: {}".format(output_a))
print("method_b: {}".format(output_b))
print('')
print(output_a['multiLine1'])
print('')
print(output_b['multiLine2'])
print('\nAll Equal?: {}'.format(payload == output_a == output_b))

Python example how to insert raw text:
Create a table in MySQL:
create table penguins(id int primary key auto_increment, msg VARCHAR(4000))
Python code:
#!/usr/bin/env python
import sqlalchemy
from sqlalchemy import text
engine = sqlalchemy.create_engine(
"mysql+mysqlconnector://yourusername:yourpassword#yourhostname.com/your_database")
db = engine.connect()
weird_string = "~!##$%^&*()_+`1234567890-={}|[]\;':\""
sql = text('INSERT INTO penguins (msg) VALUES (:msg)')
insert = db.execute(sql, msg=weird_string)
db.close()
Run it, examine output:
select * from penguins
1 ~!##$%^&*()_+`1234567890-={}|[]\;\':"
None of those characters were interpreted on insert.

Although I also think parameter binding should be used, there is also this:
>>> import MySQLdb
>>> example = r"""I don't like "special" chars ¯\_(ツ)_/¯"""
>>> example
'I don\'t like "special" chars \xc2\xaf\\_(\xe3\x83\x84)_/\xc2\xaf'
>>> MySQLdb.escape_string(example)
'I don\\\'t like \\"special\\" chars \xc2\xaf\\\\_(\xe3\x83\x84)_/\xc2\xaf'