I'm trying to get the following Python MYSQL update statement correct(With Variables):
try:
connection = mysql.connector.connect\
(host = "localhost", user = "root", passwd ="", db = "crawling")
except:
print("Keine Verbindung zum Server")
sys.exit(0)
cursor = connection.cursor()
cursor.execute("TRUNCATE meta;")
connection.commit()
cursor.execute("ALTER TABLE meta AUTO_INCREMENT =1;")
connection.commit()
for j in range(1, int(outerElements)):
for i in range(1, int(innerElements)):
partner_ID = 6
location_ID = 20
headline = driver.find_element_by_xpath("//div[#id='productList']/div["+str(j)+"]/div["+str(i)+"]/div/div[2]/h2/a").text
price = driver.find_element_by_xpath("//div[#id='productList']/div["+str(j)+"]/div["+str(i)+"]/div/div[2]/div[2]/span[2]").text[:-1]
deeplink = driver.find_element_by_xpath("//div[#id='productList']/div["+str(j)+"]/div["+str(i)+"]/div/div[2]/h2/a").get_attribute("href")
print("Header: " + headline + " | " + "Price: " + price + " | " + "Deeplink: " + deeplink + " | " + "PartnerID: " + str(partner_ID) + " | " + "LocationID: " + str(location_ID))
cursor.execute('''UPDATE meta SET (price_id, Header, Price, Deeplink, PartnerID, LocationID) \
VALUES(%s, %s, %s, %s, %s, %s)''', ['None'] + [headline] + [price] + [deeplink] + [partner_ID] + [location_ID])
connection.commit()
cursor.close()
connection.close()
When conducting the code, I´m getting the following error message:
Traceback (most recent call last):
File "C:/Users/hmattu/PycharmProjects/untitled1/localhost_crawl.py", line 97, in test_sel
VALUES(%s, %s, %s, %s, %s, %s)''', ['None'] + [headline] + [price] + [deeplink] + [partner_ID] + [location_ID])
File "C:\Python34\lib\site-packages\mysql\connector\cursor.py", line 507, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "C:\Python34\lib\site-packages\mysql\connector\connection.py", line 722, in cmd_query
result = self._handle_result(self._send_cmd(ServerCmd.QUERY, query))
File "C:\Python34\lib\site-packages\mysql\connector\connection.py", line 640, in _handle_result
raise errors.get_exception(packet)
mysql.connector.errors.ProgrammingError: 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '= (price_id, Header, Price, Deeplink, PartnerID, LocationID)
Can anybody help me out? Any feedback is appreciated.
Try to change
UPDATE meta SET (price_id, Header, Price, Deeplink, PartnerID, LocationID)
for
UPDATE meta SET (`price_id`, `Header`, `Price`, `Deeplink`, `PartnerID`, `LocationID`)
Related
I tried some website scrapping. I success scraped datas in my current db table. But I would like to INSERT into "new_table" only if records do not exist in "current table"
My code is (pipelines)
table = 'products'
table2 = 'new_products'`
def save(self, row):
cursor = self.cnx.cursor()
cursor.execute("SELECT DISTINCT product_id FROM products;")
old_ids = [row[0] for row in cursor.fetchall()]
create_query = ("INSERT INTO " + self.table +
"(rowid, date, listing_id, product_id, product_name, price, url) "
"VALUES (%(rowid)s, %(date)s, %(listing_id)s, %(product_id)s, %(product_name)s, %(price)s, %(url)s)")
cursor.execute(create_query, row)
lastRecordId = cursor.lastrowid
self.cnx.commit()
cursor.close()
print("Item saved with ID: {}" . format(lastRecordId))
if not product_id in old_ids:
create_query = ("INSERT INTO " + self.table2 +
"(rowid, date, listing_id, product_id, product_name, price, url) "
"VALUES (%(rowid)s, %(date)s, %(listing_id)s, %(product_id)s, %(product_name)s, %(price)s, %(url)s)")
it's not working and I got errors.
2022-05-06 12:26:57 [scrapy.core.scraper] ERROR: Error processing {'date': '2022-05-06 12:26:57.575507',
'listing_id': '0190199600119',
'price': '4199.00',
'product_id': '1209298',
'product_name': 'APPLE 11" Magic Türkçe Q Klavye Siyah',
'rowid': 456274953331128512,
'url': 'https://www.mediamarkt.com.tr/tr/product/APPLE%2011%22%20Magic%20T%C3%BCrk%C3%A7e%20Q%20Klavye%20Siyah-1209298.html'}
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/usr/local/lib/python3.8/dist-packages/scrapy/utils/defer.py", line 162, in f
return deferred_from_coro(coro_f(*coro_args, **coro_kwargs))
File "/root/teknosa/teknosa/pipelines.py", line 28, in process_item
self.save(dict(item))
File "/root/teknosa/teknosa/pipelines.py", line 62, in save
if not product_id in old_ids:
NameError: name 'product_id' is not defined
Saving item into db ...
I have unique product_id.
if not product_id in current_table INSERT this product_id to the "new_products"
How to make this?
Thank you.
last edit: I got this error.
2022-05-07 18:17:11 [scrapy.core.scraper] ERROR: Error processing {'date': '2022-05-07 18:17:11.902622',
'listing_id': '8713439219357',
'price': '99.00',
'product_id': '1175529',
'product_name': 'TRUST 21935 NANGA USB 3.1 Kart Okuyucu',
'rowid': -411152717288573423,
'url': 'https://www.mediamarkt.com.tr/tr/product/TRUST%2021935%20NANGA%20USB%203.1%20Kart%20Okuyucu-1175529.html'}
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/mysql/connector/connection_cext.py", line 523, in cmd_query
self._cmysql.query(query,
_mysql_connector.MySQLInterfaceError: Duplicate entry '-411152717288573423' for key 'products.rowid'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/usr/local/lib/python3.8/dist-packages/scrapy/utils/defer.py", line 162, in f
return deferred_from_coro(coro_f(*coro_args, **coro_kwargs))
File "/root/teknosa/teknosa/pipelines.py", line 28, in process_item
self.save(dict(item))
File "/root/teknosa/teknosa/pipelines.py", line 69, in save
cursor.execute(create_query, row)
File "/usr/local/lib/python3.8/dist-packages/mysql/connector/cursor_cext.py", line 269, in execute
result = self._cnx.cmd_query(stmt, raw=self._raw,
File "/usr/local/lib/python3.8/dist-packages/mysql/connector/connection_cext.py", line 528, in cmd_query
raise errors.get_mysql_exception(exc.errno, msg=exc.msg,
mysql.connector.errors.IntegrityError: 1062 (23000): Duplicate entry '-411152717288573423' for key 'products.rowid'
If you want to INSERT only if it does not exist, you don´t need to do what you are doing.There is no need to select all and then look if the one you are looking exists.
What you need is makinq a unique index for produc_id in table2
Then change your code to:
table = 'products'
table2 = 'new_products'`
def save(self, row):
create_query = ("INSERT INTO " + self.table +
"(rowid, date, listing_id, product_id, product_name, price, url) "
"VALUES (%(rowid)s, %(date)s, %(listing_id)s, %(product_id)s, %(product_name)s, %(price)s, %(url)s)")
cursor.execute(create_query, row)
lastRecordId = cursor.lastrowid
self.cnx.commit()
print("Item saved with ID: {}" . format(lastRecordId))
create_query = ("INSERT INTO " + self.table2 +
"(rowid, date, listing_id, product_id, product_name, price, url) "
"VALUES (%(rowid)s, %(date)s, %(listing_id)s, %(product_id)s, %(product_name)s, %(price)s, %(url)s) ON DUPLICATE KEY UPDATE product_id=product_id")
cursor.execute(create_query, row)
self.cnx.commit()
If you use ON DUPLICATE KEY, when it finds a duplicate row (a product_id that already exists) the system tryes to update the product_id to the same product_id, so it takes no effect.
And you can remove those commit if you set autocommit= True.
EDIT
If as you say in your comment, you need to insert in the new table only if it does not exist in your table, you can change your code like this:
You need to change the name of the variable in the line old_ids = [row[0] for row in cursor.fetchall()] because you are changing the value of your row parameter
2. Your problem is in the if statement, the product_id variable doesn´t exist, you need to change it
table = 'products'
table2 = 'new_products'`
def save(self, row):
cursor = self.cnx.cursor()
cursor.execute("SELECT DISTINCT product_id FROM products;")
old_ids = [element[0] for element in cursor.fetchall()]
create_query = ("INSERT INTO " + self.table +
"(rowid, date, listing_id, product_id, product_name, price, url) "
"VALUES (%(rowid)s, %(date)s, %(listing_id)s, %(product_id)s, %(product_name)s, %(price)s, %(url)s)")
cursor.execute(create_query, row)
lastRecordId = cursor.lastrowid
self.cnx.commit()
cursor.close()
print("Item saved with ID: {}" . format(lastRecordId))
if not row['product_id'] in old_ids:
create_query = ("INSERT INTO " + self.table2 +
"(rowid, date, listing_id, product_id, product_name, price, url) "
"VALUES (%(rowid)s, %(date)s, %(listing_id)s, %(product_id)s, %(product_name)s, %(price)s, %(url)s)")
I have bad a python script that connects to a db gets information and then uses a few variables to input this information back into the db. I have tweaked the code until it runs with no errors but in the end it doesnt actually insert anything into the db. Here is the insert code i am using
company_name = input("what is the company name?: ")
host=("")
ts = time.time()
timestamp = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
#Start DB connection
conn = mysql.connector.connect (user=dbUser,password=dbPassword,host=host,buffered=True)
cursor = conn.cursor()
#Print Database Information
select_db = ("")
cursor.execute(select_db)
api_keys = ("select * from api_keys order by API_KEYS_ID desc limit 1")
cursor.execute(api_keys)
#print(api_keys)
#for (api_keys) in cursor:
# print(api_keys[0])
#api_key_id = api_keys[0] + 1
#print(api_key_id)
api_keys_id = 170
##
add_api_key = ("INSERT INTO api_keys(api_keys_id, api_key, status, email, ip_address, filter_query, create_date, description) \
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)", (api_keys_id , api_token , 'A' , email , '*' , '(OwnerPartyID:' + ownerPartyID + ') AND (SalesStatus:' + salesStatus + ')' , timestamp , company_name + ' key'));
cursor.execute(*add_api_key)
conn.commit()
print(add_api_key)
I have a problem that I couldn't solve
I want to update data in a MysQl table but i get this error:
cursor.execute("UPDATE employees SET PhPath='~/Desktop/test/server/dataSet/%s' WHERE id=%s; ",(generate_names(UserID,1),UserID))
File "/home/chiheb/.virtualenvs/cv/local/lib/python2.7/site-packages/MySQLdb/cursors.py", line 205, in execute
self.errorhandler(self, exc, value)
File "/home/chiheb/.virtualenvs/cv/local/lib/python2.7/site-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler
raise errorclass, errorvalue
ProgrammingError: (1064, "You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'User.2.1.jpg'' WHERE id=2' at line 1")
and this is a part of my code:
data = recv_msg(conn)
data = json.loads(data)
UserName = input("enter user's name: ")
UserLastName = input("enter user's last name: ")
UserPost = input("enter user's post: ")
cursor.execute("INSERT INTO employees VALUES (NULL, %s, %s, %s, %s, NULL);",(UserName, UserLastName, UserPost, data['RasID']))
db.commit()
cursor.execute("SELECT LAST_INSERT_ID(); ")
UserIDL = cursor.fetchone()
UserID = UserIDL[0]
JL= data['Jliste']
for i in range(0,10) :
cel = json.loads(JL[i])
file_name = generate_names(UserID,i+1)
img = base64.b64decode(cel['img'])
with open(file_name,'wb') as _file:
_file.write(img)
print "image {} Received ".format(i+1)
cursor.execute("UPDATE employees SET PhPath='~/Desktop/test/server/dataSet/%s' WHERE id=%s; ",(generate_names(UserID,1),UserID))
response = "images Received "
conn.send(response)
db.commit()
The problem is that you can't do partial replacement with a parameter. Generate the path in code and only use "%s" (without the quotes) as the value.
I am getting data from the web and inserting it into mysql database by the following code. There seems to be a problem in my SQL syntax for adding records in the data table. The error message goes as follows:
Can someone help with this?
Error message:
> File "D:\Clouds\Dropbox\programming\Python\get youbike info.py", line
> 33, in <module>
> cursor.execute(insert_data) File "C:\Python34\lib\site-packages\mysql\connector\cursor.py", line 559,
> in execute
> self._handle_result(self._connection.cmd_query(stmt)) File "C:\Python34\lib\site-packages\mysql\connector\connection.py", line
> 494, in cmd_query
> result = self._handle_result(self._send_cmd(ServerCmd.QUERY, query)) File
> "C:\Python34\lib\site-packages\mysql\connector\connection.py", line
> 396, in _handle_result
> raise errors.get_exception(packet) mysql.connector.errors.ProgrammingError: 1064 (42000): You have an
> error in your SQL syntax; check the manual that corresponds to your
> MariaDB server version for the right syntax to use near '%s, %s, %s,
> %s, %s, %s, %s, %s, %s)' at line 1
My Code
import urllib.request
import gzip
import json
import mysql.connector
url = "http://data.taipei/youbike"
urllib.request.urlretrieve(url,"data.gz")
f=gzip.open('data.gz','r')
jdata = f.read()
f.close()
data = json.loads(jdata.decode('utf-8'))
stationNo =1
cnx=mysql.connector.connect(user='root', host='127.0.0.1', database='bike')
cursor = cnx.cursor()
for key,value in data["retVal"].items():
sno = value["sno"]
sna = value["sna"]
sarea = value["sarea"]
lat = value["lat"]
lng = value["lng"]
ar = value["ar"]
sareaen = value["sareaen"]
snaen = value["snaen"]
aren = value["aren"]
print("NO." + sno + " " + sna)
stationNo+=1
insert_data = ("INSERT INTO info "
"(sno, sna, sarea, lat, lng, ar, sareaen, snaen, aren) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)")
cursor.execute(insert_data)
cnx.commit()
cursor.close()
cnx.close()
you are not passing any data to the values section, the placeholder %s are going blank as there you are not passing data
insert_data = ("INSERT INTO info "
"(sno, sna, sarea, lat, lng, ar, sareaen, snaen, aren) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)")
cursor.execute(insert_data,(sno, sna, sarea, lat, lng, ar, sareaen, snaen, aren))
try this
To help fault-find, do a
print "("INSERT INTO info "
"sno, sna, sarea, lat, lng, ar, sareaen, snaen, aren) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)")
and check the values for any double-quote marks; the presence of any will definitely cause the SQL to break.
Also, by "getting data from the web" and inserting it straight into your code as you've done opens up your app to some very serious SQL-insertion attacks, so perhaps it is best to spend some time sanitizing the web input, in addition to stripping quotation marks.
I have a select query like this :
SELECT * FROM simple_tbl WHERE a_name != " " AND d_name != " " AND s_time != "0000-00-00 00:00:00" AND e_time != "0000-00-00 00:00:00"
I have to use this in a python script, I used the select inside the cursor.execute("query"), but its failing everytime.
Can anyone help me using it ?
I have used below 2 ways:
1st:
cursor.execute ("SELECT * FROM maintenance_simple_tbl WHERE acc_name != " " AND device_name != " " AND start_time != '0000-00-00 00:00:00' AND end_time != '0000-00-00 00:00:00'")
Error:
Traceback (most recent call last):
File "sync.py", line 42, in <module>
cursor.execute ("SELECT * FROM simple_tbl WHERE a_name != " " AND d_name != " " AND s_time != '0000-00-00 00:00:00' AND e_time != '0000-00-00 00:00:00'")
File "C:\Python27\lib\site-packages\MySQLdb\cursors.py", line 205, in execute
self.errorhandler(self, exc, value)
File "C:\Python27\lib\site-packages\MySQLdb\connections.py", line 36, in defaulterrorhandler
raise errorclass, errorvalue
_mysql_exceptions.ProgrammingError: (1064, "You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'AND d_name !=
ND s_time != '0000-00-00 00:00:00' AND e_time != '000' at line 1")
2nd:
cursor.execute ("SELECT * FROM simple_tbl WHERE a_name != " " AND d_name != " " AND start_time !='%s' AND e_time !='%s'" % (0000-00-00 00:00:00, 0000-00-00 00:00:00))
Error:
File "sync.py", line 43
cursor.execute ("SELECT * FROM simple_tbl WHERE a_name != " " AND d_name != " " AND s_time !='%s' AND e_time !='%s'" % (0000-00-00 00:00:00, 0000-00-00 00:00:00))
^
SyntaxError: invalid syntax
Here is how you can proceed to query MySQL in Python:
import MySQLdb as sql
dbParams = {
'host': # your database host,
'port': # your database port,
'user': # your database user,
'passwd': # your database password,
'db': # your database name
}
query = 'SELECT * FROM simple_tbl WHERE a_name != " " AND d_name != " " AND s_time != "0000-00-00 00:00:00" AND e_time != "0000-00-00 00:00:00"'
database = sql.connect(**dbParams) # initiate a connection to your database
cursor = database.cursor() # create a new cursor that you will use to query your database
cursor.execute(query) # this will execute your query
result = cursor.fetchall()
Instead of fetchall() function you could just use fetchone() if you want to retrieve the first returned row.
Although the other answer will work for executing a query and fetching the results, I think that the DictCursor object is more in line with what you would want when dealing with MySQL tables. You can use it as follows:
import MySQLdb
import MySQLdb.cursors
def get_db_cursor():
db = MySQLdb.connect(host="host",
user="user",
passwd="passwd",
db="db_name",
cursorclass=MySQLdb.cursors.DictCursor)
cur = db.cursor()
return cur
Now, you can use this function as follows to get a cursor:
cursor = get_db_cursor()
sql = "SELECT * simple_tbl WHERE a_name != ' ' AND d_name != ' ' AND s_time != '0000-00-00 00:00:00' AND e_time != '0000-00-00 00:00:00' " # Note that I replaced the double quotes with single quotes
cursor.execute(sql)
data = cursor.fetchall()
This returns a collection of dictionaries - one for each row in the resulting table. You can iterate over this collection as follows:
for row in data:
if len(row['a_name']) > 5:
# do something