for page in range(1, pages + 1):
def append_organizator(organizator, organizatorzy=[]):
organizatorzy.append(organizator)
for i in organizatorzy:
try:
query = "INSERT INTO stypendia (organizator) values(%s)"
values = []
values.append(organizatorzy.pop())
cursor.execute(query, values)
conn.commit()
except:
pass
def append_type(rodzaj, rodzaje=[]):
rodzaje.append(rodzaj)
for i in rodzaje:
try:
query = "INSERT INTO stypendia (rodzaj) values(%s)"
values = []
values.append(rodzaje.pop())
cursor.execute(query, values)
conn.commit()
except:
pass
Those are 2 functions that are inserting the data scrapped from website into the database
The program is iterating through all available pages on site. The data that's scrapped is inserted to database.
As you can see on screenshot, the title is inserted 7 times(the amount of pages), then the organizator again 7 times etc...
How can i solve this problem and have everything at same indexesdatabase ss
You need to combine the insert operations - each insert will create a new row. You should also just use the parameters without the array, they really aren't needed.
This example only handles two parameters (same as your code above). Add additional parameters as needed and adjust the insert statement
# The organization of this loop assumes the order of returned data is
# consistent: each "rodzaj" is at the same index as its "organizator"
# (as the original code assumes)
organizator = doc.find_all(class_='organizator-title')
rodzaj = doc.find_all('div', class_='fleft', string="Rodzaj:")
for i in range(min(len(organizator), len(rodzaj))):
o = organizator[i].text.strip().replace('\\n', '').replace('\\r', '')
r = rodzaji].find_next().text.strip().replace('\\n', '').replace('\\r', '')
append(o, r)
def append(organizator: str, rodzaj: str):
try:
query = "INSERT INTO stypendia (organizator, rodzaj) values(%s, %s)"
values = (organizator, rodzaj)
cursor.execute(query, values)
conn.commit()
except:
pass
Related
I am trying to create a method in python insert records into a table passing in a list of column names, and an associated list of records.
I was able to set it up where the column names populated dynamically via a for loop, but I can't figure out how to do the same thing with values because the psycopg2.executemany function relies on having %s's as placeholders.
Is it possible to have the number of %s's in the string populate dynamically via a loop? Is there another way to do this?
def load_table(dbname,table_name,fields,records):
try:
#Variable Qty Column Loop
sql_fields = []
for i in fields:
i = sql.Identifier(i)
sql_fields.append(i)
#Need similar loop to replace %s values
#Replace (%s,%s,%s) ???
#.....
#.....
sql_values = []
for i in fields:
sql_values.append('%s')
print(sql_values)
flist = sql.SQL(',').join(sql_fields)
connection, cursor = create_connection(dbname)
insert_query = sql.SQL('INSERT INTO {table_name} ({fields}) VALUES (%s,%s,%s)').format(
table_name = sql.Identifier(table_name),
fields = flist,
cursor.executemany(insert_query,records)
print('Records Loaded Successfully')
except (Exception,psycopg2.Error) as error:
print("Failed to insert record into table {error}".format(error = error))
finally:
# closing database connection.
if (connection):
close_connection(connection,cursor)
You can use sql.Placeholder, to populate the sql statement with the amount of %s-placeholders you need:
def load_table(dbname,table_name,fields,records):
con, cur = create_connection('foo')
query = sql.SQL("insert into {} ({}) values ({})").format(
sql.Identifier(table_name),
sql.SQL(', ').join(map(sql.Identifier, fields)),
sql.SQL(', ').join(sql.Placeholder() * len(fields)))
print(query.as_string(con))
if __name__ == '__main__':
dbname = '...'
table_name = 'messages'
fields = ['user_id', 'message_type', 'message_title']
records = [['12345', 'json', 'my first message'], ]
load_table(dbname,table_name,fields,records)
Output:
insert into "messages" ("user_id", "message_type", "message_title") values (%s, %s, %s)
I have a sql database Im trying to write too via a python script. That python script parses DC switches then spits it out into results_list. I then filter for the below 2 items description and interface.
The front end of the app is a flask page I have buttons assigned that will query x cabinet and do all the above. I want to send all of below data to a sql database. I also want to prevent duplicates / update table if item has changed.
right now when i run the query if the working params gets passed on to sql and the rest of the flask script is processed. When I pass params with multiple items I get the following error.
OperationalError: (1241, 'Operand should contain 1 column(s)')
working params:
params = ([('TECH2_5750',)], [('Gi1/0/7',)], [(u'10.210.44.5',)])
not working params:
params = ([('TECH2_5252',), ('TECH2_5261',), ('TECH2_5857',), ('TECH2_5278',), ('TECH2_5855',), ('TECH2_5207',), ('TECH2_6539',), ('TECH2_6363',), ('TECH2_5253',), ('TECH2_5030',), ('TECH2_6575',), ('TECH2_5839',), ('TECH2_6953',), ('TECH2_6133',), ('TECH2_5046',), ('TECH2_6152',), ('TECH2_5267',), ('TECH2_5847',), ('TECH2_5748',), ('TECH2_5104',), ('TECH2_5649',), ('TECH2_5103',)], [('Gi2/0/11',), ('Gi1/0/11',), ('Gi2/0/7',), ('Gi3/0/27',), ('Gi1/0/7',), ('Gi2/0/9',), ('Gi3/0/7',), ('Gi2/0/27',), ('Gi1/0/29',), ('Gi3/0/25',), ('Gi2/0/25',), ('Gi2/0/29',), ('Gi2/0/31',), ('Gi1/0/31',), ('Gi2/0/5',), ('Gi1/0/25',), ('Gi1/0/5',), ('Gi3/0/5',), ('Gi1/0/9',), ('Gi1/0/35',), ('Gi1/0/27',), ('Gi1/0/33',)], [(u'10.210.45.5',)])
code:
local1 = []
device1 = []
items = len(filtered_interface)
for item in range(0, items):
t = filtered_tech[item].split(' ')
hostname = t[1]
device1.append((hostname,))
t = filtered_interface[item].split(' ')
interfacelocal = t[1]
local1.append((interfacelocal,))
cab1 = []
cab1.append((cab,))
params = (device1, local1, cab1,)
connection1 = MySQLdb.connect(user="user", passwd="password", db="db",host="127.0.0.1")
mycursor = connection1.cursor()
sql = """
INSERT INTO devices (hostname, localint, cab) VALUES (%s, %s, %s)"""
mycursor.execute(sql, params)
connection1.commit()
mycursur.execute() requires that params have the same number of elements as you have placeholders in the query. Since you have 3 placeholders, params must have 3 elements.
If you want to insert multiple rows, you need to use mycursor.executemany. This will execute the query repeatedly for each element of the parameter sequence. params should then be a 2-dimensional sequence, with each element of the top-level sequence being a sequence of 3 parameters to fill in the placeholders.
params = [('TECH2_5252', 'Gi1/0/7', u'10.210.44.5'),
('TECH2_5261', 'Gi1/0/11', u'10.210.45.5'),
('TECH2_5857', 'Gi2/0/7', u'10.210.45.5'),
...]
sql = """
INSERT INTO devices (hostname, localint, cab) VALUES (%s, %s, %s)"""
mycursor.executemany(sql, params)
I have a SQL database file which contains a multitude of columns, two of which are 'GEO_ID' and 'MED_INCOME'. I am trying to retrieve just the 'MED_INCOME' column data using the associated 'GEO_ID'. Here is what I thought would work:
import sqlite3 as db
def getIncome(censusID):
conn = db.connect('census.db')
c = conn.cursor()
c.execute("SELECT 'MED_INCOME' FROM censusDbTable WHERE GEO_ID = %s" % (censusID)
response = c.fetchall()
c.close()
conn.close()
return response
id = 60014001001
incomeValue = getIncome(id)
print("incomeValue: ", incomeValue)
Which results in:
incomeValue: [('MED_INCOME',)]
I thought that I had used this method before when attempting to retrieve the data from just one column, but this method does not appear to work. If I were to instead write:
c.execute("SELECT * FROM censusDbTable WHERE GEO_ID = %s" % (censusID)
I get the full row's data, so I know the ID is in the database file.
Is there something about my syntax that is causing this request to result in an empty set?
Per #Ernxst comment, I adjusted the request to:
c.execute("SELECT MED_INCOME FROM censusDbTable WHERE GEO_ID = %s" % (censusID)
Removing the quotes around the column ID, which solved the problem.
I'm trying to insert dummy data into a mysql database.
The database structure looks like:
database name: messaround
database table name: test
table structure:
id (Primary key, auto increment)
path (varchar(254))
UPDATED 2 method below, and error.
I have a method to try to insert via:
def insert_into_db(dbcursor, table, *cols, **vals):
try:
query = "INSERT INTO {} ({}) VALUES ('{}')".format(table, ",".join(cols), "'),('".join(vals))
print(query)
dbcursor.execute(query)
dbcursor.commit()
print("inserted!")
except pymysql.Error as exc:
print("error inserting...\n {}".format(exc))
connection=conn_db()
insertstmt=insert_into_db(connection, table='test', cols=['path'], vals=['test.com/test2'])
However, this is failing saying:
INSERT INTO test () VALUES ('vals'),('cols')
error inserting...
(1136, "Column count doesn't match value count at row 1")
Can you please assist?
Thank you.
If you use your code:
def insert_into_db(dbcursor, table, *cols, **vals):
query = "INSERT INTO {} ({}) VALUES ({})".format(table,",".join(cols), ",".join(vals))
print(query)
insert_into_db('cursor_here', 'table_here', 'name', 'city', name_person='diego', city_person='Sao Paulo')
Python returns:
INSERT INTO table_here (name,city) VALUES (name_person,city_person)
Now with this other:
def new_insert_into_db(dbcursor, table, *cols, **vals):
vals2 = ''
for first_part, second_part in vals.items():
vals2 += '\'' + second_part + '\','
vals2 = vals2[:-1]
query = "INSERT INTO {} ({}) VALUES ({})".format(table,",".join(cols), vals2)
print(query)
new_insert_into_db('cursor_here', 'table_here', 'name', 'city', name_person='diego', city_person='Sao Paulo')
Python will return the correct SQL:
INSERT INTO table_here (name,city) VALUES ('diego','Sao Paulo')
Generally in Python you pass a parameterized query to the DB driver. See this example in PyMySQL's documentation; it constructs the INSERT query with placeholder characters, then calls cursor.execute() passing the query, and a tuple of the actual values.
Using parameterized queries is also recommended for security purposes, as it defeats many common SQL injection attacks.
you should print the sql statement which you've generated, that makes it a lot easier to see what's wrong.
But I guess you need quotes ' around string values for your ",".join(vals) (in case there are string values.
So your code is producing
insert into test (path,) values (test.com/test2,);
but it should produce
insert into test (`path`) values ('test.com/test2');
Otherwise try https://github.com/markuman/MariaSQL/ which makes it super easy to insert data to MariaDB/MySQL using pymysql.
Change your query as below
query = "INSERT INTO {} ({}) VALUES ('{}')".format(table, ",".join(cols), "'),('".join(vals))
As you are using join, the variable is expected to be a list but not a string
table = 'test'
cols = ['path']
vals = ['test.com/test2', 'another.com/anothertest']
print(query)
"INSERT INTO test (path) VALUES ('test.com/test2'),('another.com/anothertest')"
Update:
def insert_into_db(dbconnection=None, table='', cols=None, vals=None):
mycursor = dbconnection.cursor()
if not (dbconnection and table and cols and vals):
print('Must need all values')
quit()
try:
query = "INSERT INTO {} ({}) VALUES ('{}')".format(table, ",".join(cols), "'),('".join(vals))
mycursor.execute(query)
dbconnection.commit()
print("inserted!")
except pymysql.Error as exc:
print("error inserting...\n {}".format(exc))
connection=conn_db()
insertstmt=insert_into_db(dbconnection=connection, table='test', cols=['path'], vals=['test.com/test2'])
I have this situation where I created a method that will insert rows in database. I provide to that method columns, values and table name.
COLUMNS = [['NAME','SURNAME','AGE'],['SURNAME','NAME','AGE']]
VALUES = [['John','Doe',56],['Doe','John',56]]
TABLE = 'people'
This is how I would like to pass but it doesn't work:
db = DB_CONN.MSSQL() #method for connecting to MS SQL or ORACLE etc.
cursor = db.cursor()
sql = "insert into %s (?) VALUES(?)" % TABLE
cursor.executemany([sql,[COLUMNS[0],VALUES[0]],[COLUMNS[1],VALUES[1]]])
db.commit()
This is how it will pass query but problem is that I must have predefined column names and that's not good because what if the other list has different column sort? Than the name will be in surname and surname in name.
db = DB_CONN.MSSQL() #method for connecting to MS SQL or ORACLE etc.
cursor = db.cursor()
sql = 'insert into %s (NAME,SURNAME,AGE) VALUES (?,?,?)'
cursor.executemany(sql,[['John','Doe',56],['Doe','John',56]])
db.commit()
I hope I explained it clearly enough.
Ps. COLUMNS and VALUES are extracted from json dictionary
[{'NAME':'John','SURNAME':'Doe','AGE':56...},{'SURNAME':'Doe','NAME':'John','AGE':77...}]
if that helps.
SOLUTION:
class INSERT(object):
def __init__(self):
self.BASE_COL = ''
def call(self):
GATHER_DATA = [{'NAME':'John','SURNAME':'Doe','AGE':56},{'SURNAME':'Doe','NAME':'John','AGE':77}]
self.BASE_COL = ''
TABLE = 'person'
#check dictionary keys
for DATA_EVAL in GATHER_DATA:
if self.BASE_COL == '': self.BASE_COL = DATA_EVAL.keys()
else:
if self.BASE_COL != DATA_EVAL.keys():
print ("columns in DATA_EVAL.keys() have different columns")
#send mail or insert to log or remove dict from list
exit(403)
#if everything goes well make an insert
columns = ','.join(self.BASE_COL)
sql = 'insert into %s (%s) VALUES (?,?,?)' % (TABLE, columns)
db = DB_CONN.MSSQL()
cursor = db.cursor()
cursor.executemany(sql, [DATA_EVAL.values() for DATA_EVAL in GATHER_DATA])
db.commit()
if __name__ == "__main__":
ins = INSERT()
ins.call()
You could take advantage of the non-random nature of key-value pair listing for python dictionaries.
You should check that all items in the json array of records have the same fields, otherwise you'll run into an exception in your query.
columns = ','.join(records[0].keys())
sql = 'insert into %s (%s) VALUES (?,?,?)' % (TABLE, columns)
cursor.executemany(sql,[record.values() for record in records])
References:
https://stackoverflow.com/a/835430/5189811