Hi everyone I have problem with this query in Django
projects_name = str(kwargs['project_name']).split(',')
status = str(kwargs['status'])
list_project = tuple(projects_name)
opc_status = {'jobs_running':'running', 'jobs_pending':'pending', 'cpus_used':'cpu'}
if status in opc_status.values():
key = list(opc_status.keys())[list(opc_status.values()).index(status)] + ', entry_dt'
else:
key='*'
db = MySQLdb.connect(host='..', port=, user='..', passwd='..', db='..')
try:
cursor = db.cursor()
cursor.execute('SELECT %s FROM proj_cpus WHERE project in %s', key, list_project])
the first params of the query must be * or something like jobs_pending, entry_dt
but query return this error
tuple index out of range
Any idea about how to create the query correctly?
You could try this:
# Build a comma-separated string of all items in list_project
data_list = ', '.join([item for item in list_project])
query = 'SELECT %s FROM proj_cpus WHERE project in (%s)'
# Supply the parameters in the form of a tuple
cursor.execute(query, (key, data_list))
cursor.fetchall() will always return data in tuples like you have observed in comments, it is not because there is an issue with the query. To convert to json you could do something like the following (row_counter is just a placeholder to make sure that there is a unique key for every entry).
import json
key = '*'
data_list = ', '.join([item for item in list_project])
query = 'SELECT %s FROM proj_cpus WHERE project in (%s)'
cursor.execute(query, (key, data_list))
all_rows = cursor.fetchall()
row_headings = [header[0] for header in cursor.description]
row_counter = 0
all_rows_container = {}
for item in all_rows:
item_dict = {row_headings[x]: item[x] for x in range(len(row_headings))}
all_rows_container[row_counter] = item_dict
row_counter += 1
json_data = json.dumps(all_rows_container)
print json_data
NOTE: the above may throw IndexError if the query is not with key = '*' because I think row_headings will contain all of the schema for the table, even for values that you did not select in the query. However, it should be sufficient to demonstrate the approach and you can tailor it in the event that you pick specific columns only.
Related
I am trying to create a method in python insert records into a table passing in a list of column names, and an associated list of records.
I was able to set it up where the column names populated dynamically via a for loop, but I can't figure out how to do the same thing with values because the psycopg2.executemany function relies on having %s's as placeholders.
Is it possible to have the number of %s's in the string populate dynamically via a loop? Is there another way to do this?
def load_table(dbname,table_name,fields,records):
try:
#Variable Qty Column Loop
sql_fields = []
for i in fields:
i = sql.Identifier(i)
sql_fields.append(i)
#Need similar loop to replace %s values
#Replace (%s,%s,%s) ???
#.....
#.....
sql_values = []
for i in fields:
sql_values.append('%s')
print(sql_values)
flist = sql.SQL(',').join(sql_fields)
connection, cursor = create_connection(dbname)
insert_query = sql.SQL('INSERT INTO {table_name} ({fields}) VALUES (%s,%s,%s)').format(
table_name = sql.Identifier(table_name),
fields = flist,
cursor.executemany(insert_query,records)
print('Records Loaded Successfully')
except (Exception,psycopg2.Error) as error:
print("Failed to insert record into table {error}".format(error = error))
finally:
# closing database connection.
if (connection):
close_connection(connection,cursor)
You can use sql.Placeholder, to populate the sql statement with the amount of %s-placeholders you need:
def load_table(dbname,table_name,fields,records):
con, cur = create_connection('foo')
query = sql.SQL("insert into {} ({}) values ({})").format(
sql.Identifier(table_name),
sql.SQL(', ').join(map(sql.Identifier, fields)),
sql.SQL(', ').join(sql.Placeholder() * len(fields)))
print(query.as_string(con))
if __name__ == '__main__':
dbname = '...'
table_name = 'messages'
fields = ['user_id', 'message_type', 'message_title']
records = [['12345', 'json', 'my first message'], ]
load_table(dbname,table_name,fields,records)
Output:
insert into "messages" ("user_id", "message_type", "message_title") values (%s, %s, %s)
I am trying to create a method in python insert records into a table passing in a list of column names, and an associated list of records.
I was able to set it up where the column names populated dynamically via a for loop, but I can't figure out how to do the same thing with values because the psycopg2.executemany function relies on having %s's as placeholders.
Is it possible to have the number of %s's in the string populate dynamically via a loop? Is there another way to do this?
def load_table(dbname,table_name,fields,records):
try:
#Variable Qty Column Loop
sql_fields = []
for i in fields:
i = sql.Identifier(i)
sql_fields.append(i)
#Need similar loop to replace %s values
#Replace (%s,%s,%s) ???
#.....
#.....
sql_values = []
for i in fields:
sql_values.append('%s')
print(sql_values)
flist = sql.SQL(',').join(sql_fields)
connection, cursor = create_connection(dbname)
insert_query = sql.SQL('INSERT INTO {table_name} ({fields}) VALUES (%s,%s,%s)').format(
table_name = sql.Identifier(table_name),
fields = flist,
cursor.executemany(insert_query,records)
print('Records Loaded Successfully')
except (Exception,psycopg2.Error) as error:
print("Failed to insert record into table {error}".format(error = error))
finally:
# closing database connection.
if (connection):
close_connection(connection,cursor)
You can use sql.Placeholder, to populate the sql statement with the amount of %s-placeholders you need:
def load_table(dbname,table_name,fields,records):
con, cur = create_connection('foo')
query = sql.SQL("insert into {} ({}) values ({})").format(
sql.Identifier(table_name),
sql.SQL(', ').join(map(sql.Identifier, fields)),
sql.SQL(', ').join(sql.Placeholder() * len(fields)))
print(query.as_string(con))
if __name__ == '__main__':
dbname = '...'
table_name = 'messages'
fields = ['user_id', 'message_type', 'message_title']
records = [['12345', 'json', 'my first message'], ]
load_table(dbname,table_name,fields,records)
Output:
insert into "messages" ("user_id", "message_type", "message_title") values (%s, %s, %s)
I'm using Python 2 and have the following code:
with conn.cursor() as cursor:
info("Updating {} records".format(len(records_to_update)))
for record in records_to_update:
query = "UPDATE my_table SET "
params_setters = []
# Process all fields except wsid when updating
for index, header in enumerate(DB_COLUMNS_IN_ORDER[1:]):
if record[index] is not None:
params_setters.append("{} = '{}' ".format(header, record[index]))
query += " , ".join(params_setters)
query += " WHERE id = '{}'".format(record[0])
cursor.execute(query)
How can I use query params for escaping here and not have to do it manually in places like:
params_setters.append("{} = '{}' ".format(header, record[index]))
If I understand your question, you want to use a prepared statement. If you are using a driver where %s is used to represent a query parameter (SQLite uses ?), then:
with conn.cursor() as cursor:
info("Updating {} records".format(len(records_to_update)))
params = []
for record in records_to_update:
query = "UPDATE my_table SET "
params_setters = []
# Process all fields except wsid when updating
for index, header in enumerate(DB_COLUMNS_IN_ORDER[1:]):
if record[index] is not None:
params_setters.append("{} = %s ".format(header))
params.append(record[index])
query += " , ".join(params_setters)
query += " WHERE id = %s"
params.append(record[0])
cursor.execute(query, params)
I am pretty new in python developing. I have a long python script what "clone" a database and add additional stored functions and procedures. Clone means copy only the schema of DB.These steps work fine.
My question is about pymysql insert exection:
I have to copy some table contents into the new DB. I don't get any sql error. If I debug or print the created INSERT INTO command is correct (I've tested it in an sql editor/handler). The insert execution is correct becuse the result contain the exact row number...but all rows are missing from destination table in dest.DB...
(Ofcourse DB_* variables have been definied!)
import pymysql
liveDbConn = pymysql.connect(DB_HOST, DB_USER, DB_PWD, LIVE_DB_NAME)
testDbConn = pymysql.connect(DB_HOST, DB_USER, DB_PWD, TEST_DB_NAME)
tablesForCopy = ['role', 'permission']
for table in tablesForCopy:
with liveDbConn.cursor() as liveCursor:
# Get name of columns
liveCursor.execute("DESCRIBE `%s`;" % (table))
columns = '';
for column in liveCursor.fetchall():
columns += '`' + column[0] + '`,'
columns = columns.strip(',')
# Get and convert values
values = ''
liveCursor.execute("SELECT * FROM `%s`;" % (table))
for result in liveCursor.fetchall():
data = []
for item in result:
if type(item)==type(None):
data.append('NULL')
elif type(item)==type('str'):
data.append("'"+item+"'")
elif type(item)==type(datetime.datetime.now()):
data.append("'"+str(item)+"'")
else: # for numeric values
data.append(str(item))
v = '(' + ', '.join(data) + ')'
values += v + ', '
values = values.strip(', ')
print("### table: %s" % (table))
testDbCursor = testDbConn.cursor()
testDbCursor.execute("INSERT INTO `" + TEST_DB_NAME + "`.`" + table + "` (" + columns + ") VALUES " + values + ";")
print("Result: {}".format(testDbCursor._result.message))
liveDbConn.close()
testDbConn.close()
Result is:
### table: role
Result: b"'Records: 16 Duplicates: 0 Warnings: 0"
### table: permission
Result: b'(Records: 222 Duplicates: 0 Warnings: 0'
What am I doing wrong? Thanks!
You have 2 main issues here:
You don't use conn.commit() (which would be either be liveDbConn.commit() or testDbConn.commit() here). Changes to the database will not be reflected without committing those changes. Note that all changes need committing but SELECT, for example, does not.
Your query is open to SQL Injection. This is a serious problem.
Table names cannot be parameterized, so there's not much we can do about that, but you'll want to parameterize your values. I've made multiple corrections to the code in relation to type checking as well as parameterization.
for table in tablesForCopy:
with liveDbConn.cursor() as liveCursor:
liveCursor.execute("SELECT * FROM `%s`;" % (table))
name_of_columns = [item[0] for item in liveCursor.description]
insert_list = []
for result in liveCursor.fetchall():
data = []
for item in result:
if item is None: # test identity against the None singleton
data.append('NULL')
elif isinstance(item, str): # Use isinstance to check type
data.append(item)
elif isinstance(item, datetime.datetime):
data.append(item.strftime('%Y-%m-%d %H:%M:%S'))
else: # for numeric values
data.append(str(item))
insert_list.append(data)
testDbCursor = testDbConn.cursor()
placeholders = ', '.join(['`%s`' for item in insert_list[0]])
testDbCursor.executemany("INSERT INTO `{}.{}` ({}) VALUES ({})".format(
TEST_DB_NAME,
table,
name_of_columns,
placeholders),
insert_list)
testDbConn.commit()
From this github thread, I notice that executemany does not work as expected in psycopg2; it instead sends each entry as a single query. You'll need to use execute_batch:
from psycopg2.extras import execute_batch
execute_batch(testDbCursor,
"INSERT INTO `{}.{}` ({}) VALUES ({})".format(TEST_DB_NAME,
table,
name_of_columns,
placeholders),
insert_list)
testDbConn.commit()
How to insert data into table using python pymsql
Find my solution below
import pymysql
import datetime
# Create a connection object
dbServerName = "127.0.0.1"
port = 8889
dbUser = "root"
dbPassword = ""
dbName = "blog_flask"
# charSet = "utf8mb4"
conn = pymysql.connect(host=dbServerName, user=dbUser, password=dbPassword,db=dbName, port= port)
try:
# Create a cursor object
cursor = conn.cursor()
# Insert rows into the MySQL Table
now = datetime.datetime.utcnow()
my_datetime = now.strftime('%Y-%m-%d %H:%M:%S')
cursor.execute('INSERT INTO posts (post_id, post_title, post_content, \
filename,post_time) VALUES (%s,%s,%s,%s,%s)',(5,'title2','description2','filename2',my_datetime))
conn.commit()
except Exception as e:
print("Exeception occured:{}".format(e))
finally:
conn.close()
I'm not asking for the SHOW COLUMNS command.
I want to create an application that works similarly to heidisql, where you can specify an SQL query and when executed, returns a result set with rows and columns representing your query result. The column names in the result set should match your selected columns as defined in your SQL query.
In my Python program (using MySQLdb) my query returns only the row and column results, but not the column names. In the following example the column names would be ext, totalsize, and filecount. The SQL would eventually be external from the program.
The only way I can figure to make this work, is to write my own SQL parser logic to extract the selected column names.
Is there an easy way to get the column names for the provided SQL?
Next I'll need to know how many columns does the query return?
# Python
import MySQLdb
#===================================================================
# connect to mysql
#===================================================================
try:
db = MySQLdb.connect(host="myhost", user="myuser", passwd="mypass",db="mydb")
except MySQLdb.Error, e:
print "Error %d: %s" % (e.args[0], e.args[1])
sys.exit (1)
#===================================================================
# query select from table
#===================================================================
cursor = db.cursor ()
cursor.execute ("""\
select ext,
sum(size) as totalsize,
count(*) as filecount
from fileindex
group by ext
order by totalsize desc;
""")
while (1):
row = cursor.fetchone ()
if row == None:
break
print "%s %s %s\n" % (row[0], row[1], row[2])
cursor.close()
db.close()
cursor.description will give you a tuple of tuples where [0] for each is the column header.
num_fields = len(cursor.description)
field_names = [i[0] for i in cursor.description]
This is the same as thefreeman but more in pythonic way using list and dictionary comprehension
columns = cursor.description
result = [{columns[index][0]:column for index, column in enumerate(value)} for value in cursor.fetchall()]
pprint.pprint(result)
Similar to #James answer, a more pythonic way can be:
fields = [field_md[0] for field_md in cursor.description]
result = [dict(zip(fields,row)) for row in cursor.fetchall()]
You can get a single column with list comprehension over the result:
extensions = [row['ext'] for row in result)
or filter results using an additional if in the list comprehension:
large = [row for row in result if row['filesize'] > 1024 and row['filesize'] < 4096]
or accumulate values for filtered columns:
totalTxtSize = reduce(
lambda x,y: x+y,
filter(lambda x: x['ext'].lower() == 'txt', result)
)
I think this should do what you need (builds on the answer above) . I am sure theres a more pythony way to write it, but you should get the general idea.
cursor.execute(query)
columns = cursor.description
result = []
for value in cursor.fetchall():
tmp = {}
for (index,column) in enumerate(value):
tmp[columns[index][0]] = column
result.append(tmp)
pprint.pprint(result)
You could also use MySQLdb.cursors.DictCursor. This turns your result set into a python list of python dictionaries, although it uses a special cursor, thus technically less portable than the accepted answer. Not sure about speed. Here's the edited original code that uses this.
#!/usr/bin/python -u
import MySQLdb
import MySQLdb.cursors
#===================================================================
# connect to mysql
#===================================================================
try:
db = MySQLdb.connect(host='myhost', user='myuser', passwd='mypass', db='mydb', cursorclass=MySQLdb.cursors.DictCursor)
except MySQLdb.Error, e:
print 'Error %d: %s' % (e.args[0], e.args[1])
sys.exit(1)
#===================================================================
# query select from table
#===================================================================
cursor = db.cursor()
sql = 'SELECT ext, SUM(size) AS totalsize, COUNT(*) AS filecount FROM fileindex GROUP BY ext ORDER BY totalsize DESC;'
cursor.execute(sql)
all_rows = cursor.fetchall()
print len(all_rows) # How many rows are returned.
for row in all_rows: # While loops always make me shudder!
print '%s %s %s\n' % (row['ext'], row['totalsize'], row['filecount'])
cursor.close()
db.close()
Standard dictionary functions apply, for example, len(row[0]) to count the number of columns for the first row, list(row[0]) for a list of column names (for the first row), etc. Hope this helps!
This is only an add-on to the accepted answer:
def get_results(db_cursor):
desc = [d[0] for d in db_cursor.description]
results = [dotdict(dict(zip(desc, res))) for res in db_cursor.fetchall()]
return results
where dotdict is:
class dotdict(dict):
__getattr__ = dict.get
__setattr__ = dict.__setitem__
__delattr__ = dict.__delitem__
This will allow you to access much easier the values by column names.
Suppose you have a user table with columns name and email:
cursor.execute('select * from users')
results = get_results(cursor)
for res in results:
print(res.name, res.email)
Something similar to the proposed solutions, only the result is json with column_header : vaule for db_query ie sql.
cur = conn.cursor()
cur.execute(sql)
res = [dict((cur.description[i][0], value) for i, value in enumerate(row)) for row in cur.fetchall()]
output json example:
[
{
"FIRST_ROW":"Test 11",
"SECOND_ROW":"Test 12",
"THIRD_ROW":"Test 13"
},
{
"FIRST_ROW":"Test 21",
"SECOND_ROW":"Test 22",
"THIRD_ROW":"Test 23"
}
]
Looks like MySQLdb doesn't actually provide a translation for that API call. The relevant C API call is mysql_fetch_fields, and there is no MySQLdb translation for that
Try:
cursor.column_names
mysql connector version:
mysql.connector.__version__
'2.2.9'
You can also do this to just get the field titles:
table = cursor.description
check = 0
for fields in table:
for name in fields:
if check < 1:
print(name),
check +=1
check =0
cursor.column_names is a nice and simple one.
column_names = cursor.field_names
found an easy way of having colums like sql using pymysql and pandas
import pymysql
import pandas as pd
db = pymysql.connect(host="myhost", user="myuser", passwd="mypass", db="mydb")
query = """SELECT ext
SUM(size) as totalsize,
COUNT(*) as filecount
FROM fileindex
GROUP BY ext
ORDER BY totalsize DESC;
"""
df = pd.read_sql_query(query,db)
the DataFrame will have column names ext,totalsize,filecount by default no need to do additional stuff.
for example in my case: