I am updating a MySQL database table from a Python dictionary where the keys are the database fields. I have an insert statement that works but what I really need is INSERT ON DUPLICATE KEY UPDATE.
Here is the insert statement:
for d in r.json()['mydict'].items():
d = d[1] #the dictionary is the 2nd element in the tuple
placeholders = ', '.join(['%s'] * len(d))
columns = ', '.join(d.keys())
sql = "INSERT INTO %s ( %s ) VALUES ( %s )" % ("my_table", columns, placeholders)
c = create_connection() #from function create tables
cur = c[0] #function returns cursor
db = c[1] #function returns db
cur.execute(sql, list(d.values()))
db.commit()
db.close()
As always, any help is appreciated.
Related
I am trying to create a method in python insert records into a table passing in a list of column names, and an associated list of records.
I was able to set it up where the column names populated dynamically via a for loop, but I can't figure out how to do the same thing with values because the psycopg2.executemany function relies on having %s's as placeholders.
Is it possible to have the number of %s's in the string populate dynamically via a loop? Is there another way to do this?
def load_table(dbname,table_name,fields,records):
try:
#Variable Qty Column Loop
sql_fields = []
for i in fields:
i = sql.Identifier(i)
sql_fields.append(i)
#Need similar loop to replace %s values
#Replace (%s,%s,%s) ???
#.....
#.....
sql_values = []
for i in fields:
sql_values.append('%s')
print(sql_values)
flist = sql.SQL(',').join(sql_fields)
connection, cursor = create_connection(dbname)
insert_query = sql.SQL('INSERT INTO {table_name} ({fields}) VALUES (%s,%s,%s)').format(
table_name = sql.Identifier(table_name),
fields = flist,
cursor.executemany(insert_query,records)
print('Records Loaded Successfully')
except (Exception,psycopg2.Error) as error:
print("Failed to insert record into table {error}".format(error = error))
finally:
# closing database connection.
if (connection):
close_connection(connection,cursor)
You can use sql.Placeholder, to populate the sql statement with the amount of %s-placeholders you need:
def load_table(dbname,table_name,fields,records):
con, cur = create_connection('foo')
query = sql.SQL("insert into {} ({}) values ({})").format(
sql.Identifier(table_name),
sql.SQL(', ').join(map(sql.Identifier, fields)),
sql.SQL(', ').join(sql.Placeholder() * len(fields)))
print(query.as_string(con))
if __name__ == '__main__':
dbname = '...'
table_name = 'messages'
fields = ['user_id', 'message_type', 'message_title']
records = [['12345', 'json', 'my first message'], ]
load_table(dbname,table_name,fields,records)
Output:
insert into "messages" ("user_id", "message_type", "message_title") values (%s, %s, %s)
I need to insert multiple values into a table after checking if it doesn't exist using psycopg2.
The query am using:
WITH data(name,proj_id) as (
VALUES ('hello',123),('hey',123)
)
INSERT INTO keywords(name,proj_id)
SELECT d.name,d.proj_id FROM data d
WHERE NOT EXISTS (SELECT 1 FROM keywords u2 WHERE
u2.name=d.name AND u2.proj_id=d.proj_id)
But how to format or add the values section from tuple to ('hello',123),('hey',123) in query.
As suggested in the comment, assuming that your connection is already established as conn one of the ways would be:
from typing import Iterator, Dict, Any
def insert_execute_values_iterator(connection, keywords: Iterator[Dict[str, Any]], page_size: int = 1000) -> None:
with connection.cursor() as cursor:
psycopg2.extras.execute_values(
cursor,
""" WITH data(name,proj_id) as (VALUES %s)
INSERT INTO keywords(name,proj_id)
SELECT d.name,d.proj_id FROM data d
WHERE NOT EXISTS (SELECT 1 FROM keywords u2 WHERE
u2.name=d.name AND u2.proj_id=d.proj_id);""",
(( keyword['name'],
keyword['proj_id'] ) for keyword in keywords),
page_size=page_size)
insert_execute_values_iterator(conn,{'hello':123,'hey':123})
insert_query = """WITH data(name, proj_id) as (
VALUES (%s,%s)
)
INSERT INTO keywords(name, proj_id)
SELECT d.name,d.proj_id FROM data d
WHERE NOT EXISTS (
SELECT 1 FROM keywords u2
WHERE u2.name = d.name AND u2.proj_id = d.proj_id)"""
tuple_values = (('hello',123),('hey',123))
psycopg2.extras.execute_batch(cursor,insert_query,tuple_values)
I have two variables to insert in my table.
user_id - int
marks - float
and I am having this data for multiple users like this:
user_ids = (-,-,-,-,-,-,-) **TUPLE**
marks = (-,-,-,-,-,-,-,-) **TUPLE**
I want to insert this data into my table using executemany and I am executing this query in my flask snippet:
con = pymysql.connect(
host=host,
user=user,
password=password,
db=db,
charset=charset,
cursorclass=pymysql.cursors.DictCursor,
port=port,
)
cur = con.cursor()
percs = calcattnonull()
# percs contains list of dictionaries.
# {[<'user_id'>: <'marks'>], [<'user_id'>: <'marks'>]........}
id_ = []
perc_ = []
final = []
for perc in tqdm(percs):
id_.append(perc["user_id"])
perc_.append(perc["att_perc"])
id_ = tuple(id_)
perc_ = tuple(perc_)
final.append(id_)
final.append(perc_)
cur.executemany(
"UPDATE dream_offline_calculate SET (user_id,att_percentage) VALUES (?,?)",
final,
)
con.commit()
I am getting this error again and again:
TypeError: not all arguments converted during string formatting
Thanks in advance for helping me.
executemany takes an iterable of the same placeholders you would use when calling execute several times.
So if your original query would be
cur.execute(
"UPDATE dream_offline_calculate SET (user_id,att_percentage) VALUES (?,?)",
(user_id, att_perc),
)
the equivalent executemany would be
cur.executemany(
"UPDATE dream_offline_calculate SET (user_id,att_percentage) VALUES (?,?)",
[(user_id, att_perc)],
)
So that said, simply
cur.executemany(
"UPDATE dream_offline_calculate SET (user_id,att_percentage) VALUES (?,?)",
[(perc["user_id"], perc["att_perc"]) for perc in percs],
)
should do the trick.
I have this situation where I created a method that will insert rows in database. I provide to that method columns, values and table name.
COLUMNS = [['NAME','SURNAME','AGE'],['SURNAME','NAME','AGE']]
VALUES = [['John','Doe',56],['Doe','John',56]]
TABLE = 'people'
This is how I would like to pass but it doesn't work:
db = DB_CONN.MSSQL() #method for connecting to MS SQL or ORACLE etc.
cursor = db.cursor()
sql = "insert into %s (?) VALUES(?)" % TABLE
cursor.executemany([sql,[COLUMNS[0],VALUES[0]],[COLUMNS[1],VALUES[1]]])
db.commit()
This is how it will pass query but problem is that I must have predefined column names and that's not good because what if the other list has different column sort? Than the name will be in surname and surname in name.
db = DB_CONN.MSSQL() #method for connecting to MS SQL or ORACLE etc.
cursor = db.cursor()
sql = 'insert into %s (NAME,SURNAME,AGE) VALUES (?,?,?)'
cursor.executemany(sql,[['John','Doe',56],['Doe','John',56]])
db.commit()
I hope I explained it clearly enough.
Ps. COLUMNS and VALUES are extracted from json dictionary
[{'NAME':'John','SURNAME':'Doe','AGE':56...},{'SURNAME':'Doe','NAME':'John','AGE':77...}]
if that helps.
SOLUTION:
class INSERT(object):
def __init__(self):
self.BASE_COL = ''
def call(self):
GATHER_DATA = [{'NAME':'John','SURNAME':'Doe','AGE':56},{'SURNAME':'Doe','NAME':'John','AGE':77}]
self.BASE_COL = ''
TABLE = 'person'
#check dictionary keys
for DATA_EVAL in GATHER_DATA:
if self.BASE_COL == '': self.BASE_COL = DATA_EVAL.keys()
else:
if self.BASE_COL != DATA_EVAL.keys():
print ("columns in DATA_EVAL.keys() have different columns")
#send mail or insert to log or remove dict from list
exit(403)
#if everything goes well make an insert
columns = ','.join(self.BASE_COL)
sql = 'insert into %s (%s) VALUES (?,?,?)' % (TABLE, columns)
db = DB_CONN.MSSQL()
cursor = db.cursor()
cursor.executemany(sql, [DATA_EVAL.values() for DATA_EVAL in GATHER_DATA])
db.commit()
if __name__ == "__main__":
ins = INSERT()
ins.call()
You could take advantage of the non-random nature of key-value pair listing for python dictionaries.
You should check that all items in the json array of records have the same fields, otherwise you'll run into an exception in your query.
columns = ','.join(records[0].keys())
sql = 'insert into %s (%s) VALUES (?,?,?)' % (TABLE, columns)
cursor.executemany(sql,[record.values() for record in records])
References:
https://stackoverflow.com/a/835430/5189811
I have that query in a python program:
And i should create a multidimensional array (if it possible) or four arrays from this query for every column from the query.
Can you suggest an elegant way to solve it?
conn = #connection to the server
cursor=conn.cursor()
query = (" select id, name, phone, city from guest")
cursor.execute(query)
results = cursor.fetchall
for i in results:
print i
cursor.close()
conn.close()
Not elegant but it may assist to unravel the mysterious Python Connector Cursor Class and transfers the list of tuples (see Copperfield comment) with the data from the query, into a list (phoneList) of dictionaries (entries) with details of each entry in the database, that might be easier to work with in your python script:
# ref: https://dev.mysql.com/doc/connector-python/en/connector-python-api-mysqlcursor.html
import mysql.connector
db = 'test'
table = 'phonebook'
phoneList = []
drop_table = ("DROP TABLE IF EXISTS {};").format(table)
# By default, the starting value for AUTO_INCREMENT is 1, and it will increment by 1 for each new record.
# To let the AUTO_INCREMENT sequence start with another value, use the following SQL statement:
# ALTER TABLE phonebook AUTO_INCREMENT=100;
create_table = ("CREATE TABLE {} ("
"id int NOT NULL AUTO_INCREMENT,"
"name varchar(30) NOT NULL,"
"phone varchar(30) NOT NULL,"
"city varchar(30) NOT NULL,"
"PRIMARY KEY (id))"
" ENGINE=InnoDB DEFAULT CHARSET=latin1;").format(table)
Names = {'Bill':{'phone':'55123123','city':'Melbourne'},
'Mary':{'phone':'77111123','city':'Sydney'},
'Sue':{'phone':'55888123','city':'Melbourne'},
'Harry':{'phone':'77777123','city':'Sydney'},
'Fred':{'phone':'88123444','city':'Yongala'},
'Peter':{'phone':'55999123','city':'Melbourne'}}
cnx = mysql.connector.connect(user='mysqluser', password='xxxx',host='127.0.0.1',database=db)
cursor = cnx.cursor(dictionary=True) # key to using **row format
cursor.execute(drop_table)
cursor.execute(create_table)
# populate db
for name,detail in dict.items(Names):
sql = ("INSERT INTO {} (name,phone,city) VALUES ('{}','{}','{}')".format(table,name,detail['phone'],detail['city']))
cursor.execute(sql)
sql = ("SELECT id,name,phone,city FROM {}".format(table))
cursor.execute(sql)
for row in cursor:
print("{id} {name} {phone} {city}".format(**row))
phoneList.append(row)
print phoneList[0]['name'],phoneList[0]['city']
print phoneList[3]['name'],phoneList[3]['phone']
for entries in phoneList: # list of dictionaries
print entries['name'],entries
for entries in phoneList:
for k,v in dict.items(entries):
print k,v
print "\n"
cnx.close()