I have the following problem: I need a dynamic create statement, depending on what attributes my object has.
its following object:
class Table:
columns = []
def __init__(self, name, columns):
self.columns = columns
self.name = name
def columnsNumber(self) -> int:
return self.columns.__len__()
this is what the insert looks like:
sql = "INSERT INTO tableOverview (tableName, columns, datum) VALUES(%s, %s, CURRENT_TIMESTAMP);"
val = (table.name, table.columns.__len__())
await cursor.execute(sql, (val))
for x in table.columns:
sql = "ALTER TABLE %s ADD COLUMN %s VARCHAR(100) UNIQUE " % (table.name,x)
await cursor.execute(sql)
now I don't know, how to prevent a SQL injection.
For the ALTER TABLE statements you can quote the identifier names with backticks as described here.
for x in table.columns:
sql = "ALTER TABLE `%s` ADD COLUMN `%s` VARCHAR(100) UNIQUE " % (table.name,x)
await cursor.execute(sql)
In the insert statement, the code is already correctly using parameter substitution to ensure the inserted values are correctly quoted.
sql = "INSERT INTO tableOverview (tableName, columns, datum) VALUES(%s, %s, CURRENT_TIMESTAMP);"
val = (table.name, table.columns.len())
await cursor.execute(sql, val)
Related
I stepped over an problem while implementing a Database into my Python project.
I'm creating a new Table with the following Code:
mycursor = mydb.cursor()
sql = f"CREATE TABLE IF NOT EXISTS _{self.client_id} (tour_date DATE, tour_distance INT, tour_duration INT, tour_elevation_up INT, tour_elevation_down INT, tour_map_image TEXT, tour_name TEXT, tour_sport TEXT, tour_start_point TEXT, tour_type TEXT)"
mycursor.execute(sql)
mydb.commit()
I'm iterating over my Data and want to past it into the Table. But I won't want that if an entry already exists in the table it adds the same data again.
This is my code I currently have to Insert into my Table:
mycursor = mydb.cursor()
sql = f"INSERT INTO _{self.client_id} (tour_date, tour_distance, tour_duration, tour_elevation_up, tour_elevation_down, tour_map_image, tour_name, tour_sport, tour_start_point, tour_type) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) "
val = (TourDate, TourDistance, TourDuration, TourElevation_up, TourElevation_down, TourMap_image, TourName, TourSport, TourStart_point, TourType)
mycursor.execute(sql, val)
mydb.commit()
So my question is how can I check if a entry already exists in the Table and then avoiding creating a duplicate?
you can "select count() from your_table_name where client_id='current_id'
if count() return int that is greater than 0, you should not insert it into the databse.
First - avoid to use TEXT without it necessary
Second - create table with necessary indexes:
CREATE TABLE IF NOT EXISTS _{self.client_id} (
tour_date DATE,
tour_distance INT,
tour_duration INT,
tour_elevation_up INT,
tour_elevation_down INT,
tour_map_image TEXT,
tour_name VARCHAR(64) PRIMARY KEY,
tour_sport VARCHAR(64),
tour_start_point VARCHAR(64),
tour_type VARCHAR(64)
);
Third - use INSERT IGNORE ... statement for prevent duplicates
I am trying to use python to insert a record into a MySQL database and then update that record. To do this I have created 2 functions:
def insert_into_database():
query = "INSERT INTO pcf_dev_D.users(user_guid,username) VALUES (%s, %s) "
data = [('1234', 'user1234')]
parser = ConfigParser()
parser.read('db/db_config.ini')
db = {}
section = 'mysql'
if parser.has_section(section):
items = parser.items(section)
for item in items:
db[item[0]] = item[1]
else:
raise Exception('{0} not found in the {1} file'.format(section, filename))
try:
conn = MySQLConnection(**db)
cursor = conn.cursor()
cursor.executemany(query, data)
conn.commit()
except Error as e:
print('Error:', e)
finally:
# print("done...")
cursor.close()
conn.close()
This works fine and inserts 1234, user1234 into the db.
Now I want to update this particular user's username to '5678', so I have created another function:
def upsert_into_database():
query = "INSERT INTO pcf_dev_D.users(user_guid,username) " \
"VALUES (%s, %s) ON DUPLICATE KEY UPDATE username='%s'"
data = [('1234', 'user1234', 'user5678')]
parser = ConfigParser()
parser.read('db/db_config.ini')
db = {}
section = 'mysql'
if parser.has_section(section):
items = parser.items(section)
for item in items:
db[item[0]] = item[1]
else:
raise Exception('{0} not found in the {1} file'.format(section, 'db/db_config.ini'))
try:
conn = MySQLConnection(**db)
cursor = conn.cursor()
cursor.executemany(query, data)
conn.commit()
except Error as e:
print('Error:', e)
finally:
# print("done...")
cursor.close()
conn.close()
Which produces the following error:
Error: Not all parameters were used in the SQL statement
What's interesting is if I modify query and data to be:
query = "INSERT INTO pcf_dev_D.users(user_guid,username) " \
"VALUES (%s, %s) ON DUPLICATE KEY UPDATE username='user5678'"
data = [('1234', 'user1234')]
Then python updates the record just fine...what am I missing?
You included the 3rd parameter within single quotes in the update clause, therefore it is interpreted as part of a string, not as a placeholder for parameter. You must not enclose a parameter by quotes:
query = "INSERT INTO pcf_dev_D.users(user_guid,username) " \
"VALUES (%s, %s) ON DUPLICATE KEY UPDATE username=%s"
UPDATE
If you want to use the on duplicate key update clause with a bulk insert (e.g. executemany()), then you should not provide any parameters in the update clause because you can only have one update clause in the bulk insert statement. Use the values() function instead:
query = "INSERT INTO pcf_dev_D.users(user_guid,username) " \
"VALUES (%s, %s) ON DUPLICATE KEY UPDATE username=VALUES(username)"
In assignment value expressions in the ON DUPLICATE KEY UPDATE clause, you can use the VALUES(col_name) function to refer to column values from the INSERT portion of the INSERT ... ON DUPLICATE KEY UPDATE statement. In other words, VALUES(col_name) in the ON DUPLICATE KEY UPDATE clause refers to the value of col_name that would be inserted, had no duplicate-key conflict occurred. This function is especially useful in multiple-row inserts. The VALUES() function is meaningful only in the ON DUPLICATE KEY UPDATE clause or INSERT statements and returns NULL otherwise.
I have this situation where I created a method that will insert rows in database. I provide to that method columns, values and table name.
COLUMNS = [['NAME','SURNAME','AGE'],['SURNAME','NAME','AGE']]
VALUES = [['John','Doe',56],['Doe','John',56]]
TABLE = 'people'
This is how I would like to pass but it doesn't work:
db = DB_CONN.MSSQL() #method for connecting to MS SQL or ORACLE etc.
cursor = db.cursor()
sql = "insert into %s (?) VALUES(?)" % TABLE
cursor.executemany([sql,[COLUMNS[0],VALUES[0]],[COLUMNS[1],VALUES[1]]])
db.commit()
This is how it will pass query but problem is that I must have predefined column names and that's not good because what if the other list has different column sort? Than the name will be in surname and surname in name.
db = DB_CONN.MSSQL() #method for connecting to MS SQL or ORACLE etc.
cursor = db.cursor()
sql = 'insert into %s (NAME,SURNAME,AGE) VALUES (?,?,?)'
cursor.executemany(sql,[['John','Doe',56],['Doe','John',56]])
db.commit()
I hope I explained it clearly enough.
Ps. COLUMNS and VALUES are extracted from json dictionary
[{'NAME':'John','SURNAME':'Doe','AGE':56...},{'SURNAME':'Doe','NAME':'John','AGE':77...}]
if that helps.
SOLUTION:
class INSERT(object):
def __init__(self):
self.BASE_COL = ''
def call(self):
GATHER_DATA = [{'NAME':'John','SURNAME':'Doe','AGE':56},{'SURNAME':'Doe','NAME':'John','AGE':77}]
self.BASE_COL = ''
TABLE = 'person'
#check dictionary keys
for DATA_EVAL in GATHER_DATA:
if self.BASE_COL == '': self.BASE_COL = DATA_EVAL.keys()
else:
if self.BASE_COL != DATA_EVAL.keys():
print ("columns in DATA_EVAL.keys() have different columns")
#send mail or insert to log or remove dict from list
exit(403)
#if everything goes well make an insert
columns = ','.join(self.BASE_COL)
sql = 'insert into %s (%s) VALUES (?,?,?)' % (TABLE, columns)
db = DB_CONN.MSSQL()
cursor = db.cursor()
cursor.executemany(sql, [DATA_EVAL.values() for DATA_EVAL in GATHER_DATA])
db.commit()
if __name__ == "__main__":
ins = INSERT()
ins.call()
You could take advantage of the non-random nature of key-value pair listing for python dictionaries.
You should check that all items in the json array of records have the same fields, otherwise you'll run into an exception in your query.
columns = ','.join(records[0].keys())
sql = 'insert into %s (%s) VALUES (?,?,?)' % (TABLE, columns)
cursor.executemany(sql,[record.values() for record in records])
References:
https://stackoverflow.com/a/835430/5189811
I run the following statement through psycopg2 (which works):
self.cursor.execute( """INSERT INTO """ + self.config.schema + """.parcel (
id,geometry) VALUES (%s, ST_GeomFromGML(%s))""", (self.id, self.geometry)
But now I need to introduce some dynamics and would like to try something like this:
if multi:
mygeom = "ST_Multi(ST_GeomFromGml(" + self.geometry + "))"
else
mygeom = "ST_GeomFromGml(" + self.geometry + ")"
self.cursor.execute( """INSERT INTO """ + self.config.schema + """.parcel (
id,geometry) VALUES (%s, %s)""", (self.id, mygeom)
Of course this will fail big time since it will parse the entire function as an escaped string. Is there anyone that has any experience of how to resolve database functions to parse them dynamically in the execute statement or is this impossible?
self.cursor.execute("INSERT INTO {}.parcel (id,geometry) VALUES (%s, %s)".format( self.config.schema), (self.id, mygeom))
if multi:
mygeom = "ST_Multi(ST_GeomFromGml(%s))"
else:
mygeom = "ST_GeomFromGml(%s)"
self.cursor.execute( """INSERT INTO {}.parcel (
id,geometry) VALUES (%s, {})""" .format(self.config.schema, mygeom) , (self.id, self.geometry) )
I inserted the schema and function with a string format method. I also inserted self.geometry in a safe way.
I am having trouble inserting a record into a MySQL database from python. This is what I am doing.
def testMain2():
conn = MySQLdb.connect(charset='utf8', host="localhost", user="root", passwd="root", db="epf")
cursor = conn.cursor()
tableName = "test_table"
columnsDef = "(export_date BIGINT, storefront_id INT, genre_id INT, album_id INT, album_rank INT)"
exStr = """CREATE TABLE %s %s""" % (tableName, columnsDef)
cursor.execute(exStr)
#Escape the record
values = ["1305104402172", "12", "34", "56", "78"]
values = [conn.literal(aField) for aField in values]
stringList = "(%s)" % (", ".join(values))
columns = "(export_date, storefront_id, genre_id, album_id, album_rank)"
insertStmt = """INSERT INTO %s %s VALUES %s""" % (tableName, columns, stringList)
cursor.execute(insertStmt)
cursor.close()
conn.close()
The table is created however nothing is in the table. I can run the INSERT statement successfully in Terminal with the same credentials.
Any suggestions on what I may be doing wrong?
You haven't committed the transaction.
conn.commit()
(The MySQLdb library sets autocommit to False when connecting to MySQL. This means that you need to manually call commit or your changes will never make it into the database.)