I need to update/insert rows to MySQL database using the data from Postgres DB.So here is the script which i'm using but getting the below error while i schedule this in Jenkins.
Can anyone please guide on what i can do/change to rectify this.
File "signup.py", line 80, in <module>
11:59:27 cur_msql_1.execute(msql_insert_1, row)
11:59:27 File "/usr/local/lib/python3.5/dist-packages/MySQLdb/cursors.py", line 209, in execute
11:59:27 res = self._query(query)
11:59:27 File "/usr/local/lib/python3.5/dist-packages/MySQLdb/cursors.py", line 315, in _query
11:59:27 db.query(q)
11:59:27 File "/usr/local/lib/python3.5/dist-packages/MySQLdb/connections.py", line 239, in query
11:59:27 _mysql.connection.query(self, query)
11:59:27 MySQLdb._exceptions.ProgrammingError: (1064, 'You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near \'"timestamp", ip, store_id, confirmed_at) SELECT \'user123#gmail.com\', 15463\' at line 2')
11:59:27 Build step 'Execute shell' marked build as failure
11:59:27 Finished: FAILURE
Below is the entire code:
import psycopg2
import os
import time
import MySQLdb
import sys
from pprint import pprint
from datetime import datetime
from utils.config import Configuration as Config
from utils.postgres_helper import get_connection
from utils.utils import get_global_config
# MySQLdb connection
try:
source_host = 'magento'
conf = get_global_config()
cnx_msql = MySQLdb.connect(host=conf.get(source_host, 'host'),
user=conf.get(source_host, 'user'),
passwd=conf.get(source_host, 'password'),
port=int(conf.get(source_host, 'port')),
db=conf.get(source_host, 'db'))
print('Magento MySQL DB Connected')
except mysql.connector.Error as e:
print ("MYSQL: Unable to connect!", e.msg)
sys.exit(1)
# Postgresql connection
try:
cnx_psql = get_connection(get_global_config(), 'pg_dwh')
print('DWH PostgreSQL DB Connected')
except psycopg2.Error as e:
print('PSQL: Unable to connect!\n{0}').format(e)
sys.exit(1)
# Cursors initializations
cur_msql = cnx_msql.cursor()
cur_msql_1 = cnx_msql.cursor()
cur_psql = cnx_psql.cursor()
cur_psql_1 = cnx_psql.cursor()
now = time.strftime('%Y-%m-%d %H:%M:%S')
##################################################################################
update_sql_base="""select gr.email from unsubscribed_contacts gr
INNER JOIN subscriber sn on sn.email=gr.email"""
msql_update_1="""UPDATE subscriber SET status=3,timestamp=CAST(TO_CHAR(now(),'YYYY-MM-DD HH24:MI:SS') AS TIMESTAMP) WHERE email='%s'"""
msql_update_2="""UPDATE n_subscriber SET subscriber_status=3,change_status_at=CAST(TO_CHAR(now(),'YYYY-MM-DD HH24:MI:SS') AS TIMESTAMP)
WHERE subscriber_email='%s';"""
cur_psql.execute(update_sql_base)
for row in cur_psql:
email=row[0]
cur_msql.execute(msql_update_1 %email)
cnx_msql.commit()
cur_msql.execute(msql_update_2 %email)
cnx_msql.commit()
##################################################################################
insert_sql_base="""select gr.email,c.customer_id,'',3,'',CAST(TO_CHAR(now(),'YYYY-MM-DD HH24:MI:SS') AS TIMESTAMP),'','',CAST(TO_CHAR(now(),'YYYY-MM-DD HH24:MI:SS') AS TIMESTAMP)
from unsubscribed_contacts gr
LEFT JOIN n_subscriber sn on sn.email=gr.email
LEFT JOIN customers_2 c on c.customer_email=gr.email
WHERE sn.email IS NULL"""
msql_insert="""INSERT INTO n_subscriber(
email, customer_id, options, status, confirm_code, "timestamp", ip, store_id, confirmed_at) SELECT """
msql_insert_1="""INSERT INTO n_subscriber(
email, customer_id, options, status, confirm_code, "timestamp", ip, store_id, confirmed_at) SELECT %s, %s, %s, %s, %s, %s, %s, %s, %s"""
cur_psql_1.execute(insert_sql_base)
for row in cur_psql_1:
print(msql_insert_1)
cur_msql_1.execute(msql_insert_1, row)
cnx_msql.commit()
## Closing cursors'
cur_msql.close()
cur_psql.close()
cur_psql_1.close()
cur_msql_1.close()
## Closing database connections
cnx_msql.close()
cnx_psql.close()
Python : 3.5
PostgreSQL: Version 11
The main problem is wrong syntax(cur_msql_1.execute(msql_insert_1, row)). Just trying to explain using a few tables:
create table subscriber
(
customer_id int null,
email varchar(100) null,
timestamp int null
);
INSERT INTO subscriber (customer_id, email, timestamp) VALUES (1, 'test1#gmail.com', 1591187277);
INSERT INTO subscriber (customer_id, email, timestamp) VALUES (2, 'test2#gmail.com', 1591187303);
create table n_subscriber
(
customer_id int null,
email varchar(100) null,
timestamp int null
);
in your case it works something like this:
import MySQLdb
db = MySQLdb.connect(...)
cursor = db.cursor()
cursor.execute("SELECT customer_id, email, timestamp FROM subscriber")
for row in cursor:
cursor.execute("""INSERT INTO n_subscriber(customer_id, email, "timestamp") SELECT %s, %s, %s""", row)
db.commit()
MySQLdb._exceptions.ProgrammingError: (1064, 'You have an error in
your SQL syntax; check the manual that corresponds to your MySQL
server version for the right syntax to use near \'"timestamp") SELECT
1, \'test1#gmail.com\', 1591187277\' at line 1')
Correct syntax:
cursor.execute("INSERT INTO n_subscriber(customer_id, email, timestamp) VALUES (%s, %s, %s)", row)
Also you can do it using executemany():
cursor = db.cursor()
cursor.execute("SELECT customer_id, email, timestamp FROM subscriber")
data = cursor.fetchall()
cursor.executemany("INSERT INTO n_subscriber(customer_id, email, timestamp) VALUES (%s, %s, %s)", data)
db.commit()
Hope this helps.
Related
I've got a problem when trying to insert a json, which was converted from a python object with json.dumps, into a MySQL database. The connection to the database is working with another python file. I've already tried to just insert values, which was working, but with the json file it's not working.
My Python file:
import json
import dbConnection
cur = dbConnection.cursor
cnx = dbConnection.conn
DEVICES = {
"id": "1",
"isPoweredOn": "True",
"os": "Linux"
}
j = json.dumps(DEVICES)
print(j)
sql = "INSERT INTO DEVICES (id, isPoweredOn, os) VALUES (%s, %s, %s)"
val = (json.dumps(DEVICES))
cur.execute(sql, val)
cnx.commit()
print(cur.rowcount, "record inserted.")
Error code I get, when trying to execute:
"id": "1", "isPoweredOn": "True", "os": "Linux"}
Traceback (most recent call last):
File "dbInit.py", line 22, in <module>
cur.execute(sql, val)
File "/home/silvan/.virtualenvs/pyproj1/lib/python3.8/site-packages/mysql/connector/cursor.py", line 551, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "/home/silvan/.virtualenvs/pyproj1/lib/python3.8/site-packages/mysql/connector/connection.py", line 490, in cmd_query
result = self._handle_result(self._send_cmd(ServerCmd.QUERY, query))
File "/home/silvan/.virtualenvs/pyproj1/lib/python3.8/site-packages/mysql/connector/connection.py", line 395, in _handle_result
raise errors.get_exception(packet)
mysql.connector.errors.ProgrammingError: 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '%s, %s, %s)' at line 1
My CREATE TABLE code:
CREATE TABLE DEVICES(id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, isPoweredOn BOOLEAN NOT NULL, os VARCHAR(50) NOT NULL);
Thanks for any help in advance!
You need to json.loads(j) and assign it to a variable, then you can access the values properly.
Try :
import json
import dbConnection
cur = dbConnection.cursor
cnx = dbConnection.conn
DEVICES = {
"id": "1",
"isPoweredOn": False ,
"os": "Linux"
}
j = json.dumps(DEVICES)
values = json.loads(j)
'''
# Quick debugging
print(j , type(j))
print(values , type(values))
print(values['isPoweredOn'])
'''
sql = "INSERT INTO DEVICES (id, isPoweredOn, os) VALUES (%s, %s, %s)"
val = ( '' , values['isPoweredOn'] , values['os'])
cur.execute(sql, val)
cnx.commit()
print(cur.rowcount, "record inserted.")
Also since you defined id to be INT AUTO_INCREMENT PRIMARY KEY NOT NULL it , you can't insert device id wich is values['id'] to id column, you can alter DEVICES table and create a new column called device_id for storing the device id if you need really need to store values['id']
Firstly, cast the DEVICES to dict, then Here's the format.
sql = "INSERT INTO DEVICES (`id`, `isPoweredOn`, `os`) VALUES (%(id)s, %(isPoweredOn)s, %(os)s)"
Then Execute it :
try:
cur.execute(sql, DEVICES)
cnx.commit()
except error:
print(error)
Cheers!!
I can not execute a Python file to import CSV file to MySQL table.
import csv
import mysql.connector
mydb = mysql.connector.connect(
host='localhost',
user='root',
passwd='',
database='ricoh_oms'
)
cursor = mydb.cursor()
csv_data = csv.reader(open("slv_internal.csv"))
for row in csv_data:
cursor.execute("INSERT INTO slv_internal (GID, FullName, CostCenter) VALUES (%s, %s, %s)")
mydb.commit()
cursor.close()
print("Done")
I am newbie and dont understand the meaning of marker %s in the error. Thansk for your help. The error is:
Error:
"mysql.connector.errors.ProgrammingError: 1064 (42000): You have an
error in your SQL syntax; check the manual that corresponds to your
MySQL server version for the right syntax to use near '%s, %s, %s)' at
line 1"
you are trying to Execute Parameterized Query
Example:
insert_stmt = (
"INSERT INTO employees (emp_no, first_name, last_name, hire_date) "
"VALUES (%s, %s, %s, %s)"
)
data = (2, 'Jane', 'Doe', datetime.date(2012, 3, 23))
cursor.execute(insert_stmt, data)
in code you are missing parameters:
cursor.execute("INSERT INTO slv_internal (GID, FullName, CostCenter) VALUES (%s, %s, %s)", ("value1","value2","value3"))
some documentation:
https://dev.mysql.com/doc/connector-python/en/connector-python-api-mysqlcursor-execute.html
I have found some other questions that have a similar error to what I am getting, but have not been able to figure out how to resolve this based on the answers. I am trying to import an excel file into SQL Server with the help of python. This is the code I wrote:
import pandas as pd
import numpy as np
import pandas.io.sql
import pyodbc
import xlrd
server = "won't disclose private info"
db = 'private info'
conn = pyodbc.connect('DRIVER={SQL Server};SERVER=' + Server + ';DATABASE=' +
db + ';Trusted_Connection=yes')
cursor = conn.cursor()
book = xlrd.open_workbook("Daily Flash.xlsx")
sheet = book.sheet_by_name("Sheet1")
query1 = """CREATE TABLE [LEAF].[MK] ([LEAF][Lease_Number] varchar(255),
[LEAF][Start_Date] varchar(255), [LEAF][Report_Status] varchar(255), [LEAF]
[Status_Date] varchar(255), [LEAF][Current_Status] varchar(255), [LEAF]
[Sales_Rep] varchar(255), [LEAF][Customer_Name] varchar(255),[LEAF]
[Total_Finance] varchar(255),
[LEAF][Rate_Class] varchar(255) ,[LEAF][Supplier_Name] varchar(255) ,[LEAF]
[DecisionStatus] varchar(255))"""
query = """INSERT INTO [LEAF].[MK] (Lease_Number, Start_Date, Report_Status,
Status_Date, Current_Status, Sales_Rep, Customer_Name,Total_Finance,
Rate_Class,Supplier_Name,DecisionStatus) VALUES (%s, %s, %s, %s, %s, %s, %s,
%s, %s, %s, %s)"""
for r in range(1, sheet.nrows):
Lease_Number = sheet.cell(r,0).value
Start_Date = sheet.cell(r,1).value
Report_Status = sheet.cell(r,2).value
Status_Date = sheet.cell(r,3).value
Current_Status= sheet.cell(r,4).value
Sales_Rep = sheet.cell(r,5).value
Customer_Name = sheet.cell(r,6).value
Total_Financed= sheet.cell(r,7).value
Rate_Class = sheet.cell(r,8).value
Supplier_Name = sheet.cell(r,9).value
DecisionStatus= sheet.cell(r,10).value
values = (Lease_Number, Start_Date, Report_Status, Status_Date,
Current_Status, Sales_Rep, Customer_Name, Total_Financed, Rate_Class,
Supplier_Name, DecisionStatus)
cursor.execute(query1)
cursor.execute(query, values)
database.commit()
database.close()
database.commit()
The error message I get is:
ProgrammingError Traceback (most recent call last)
<ipython-input-24-c525ebf0af73> in <module>()
16
17 # Execute sql Query
---> 18 cursor.execute(query, values)
19
20 # Commit the transaction
ProgrammingError: ('The SQL contains 0 parameter markers, but 11 parameters
were supplied', 'HY000')
Can someone please explain the problem to me and how I can fix it? Thank you!
Update:
I have gotten that error message to go away based on the comments below. I modified my query also because the table into which I am trying to insert values into was not previously created, so I updated my code in an attempt to create it.
However, now I am getting the error message:
ProgrammingError: ('42000', '[42000] [Microsoft][ODBC SQL Server Driver][SQL
Server]The specified schema name "dbo" either does not exist or you do not
have permission to use it. (2760) (SQLExecDirectW)')
I tried changing that slightly by writing CREATE [HELLO][MK] instead of just CREATE MK but that tells me that MK is already in the database... What steps should I take next?
Based on the conversation we had in our chat, here are a few takeaways:
After executing your CREATE TABLE query, make sure to commit it immediately before running any subsequent INSERT queries.
Use error catching for cases when the table already exists in the database. You asked that if you wanted to import more data to the table, would the script still run. The answer is no, since Python will throw an exception at cursor.execute(query1).
If you want to validate whether your insert operations were successful, you can do a simple record count check.
EDIT
Yesterday, when I had #mkheifetz test my code out, he caught a minor bug where the validation check would return False, and the reason was because the database already had existing records, so when comparing against only the current data being imported, the validation would fail. Therefore, as a solution to address the bug, I have modified the code again.
Below is how I would modify your code:
import pandas as pd
import numpy as np
import seaborn as sns
import scipy.stats as stats
import matplotlib.pyplot as plt
import pandas.io.sql
import pyodbc
import xlrd
server = 'XXXXX'
db = 'XXXXXdb'
# create Connection and Cursor objects
conn = pyodbc.connect('DRIVER={SQL Server};SERVER=' + server + ';DATABASE=' + db + ';Trusted_Connection=yes')
cursor = conn.cursor()
# read data
data = pd.read_excel('Flash Daily Apps through 070918.xls')
# rename columns
data = data.rename(columns={'Lease Number': 'Lease_Number',
'Start Date': 'Start_Date',
'Report Status': 'Report_Status',
'Status Date': 'Status_Date',
'Current Status': 'Current_Status',
'Sales Rep': 'Sales_Rep',
'Customer Name': 'Customer_Name',
'Total Financed': 'Total_Financed',
'Rate Class': 'Rate_Class',
'Supplier Name': 'Supplier_Name'})
# export
data.to_excel('Daily Flash.xlsx', index=False)
# Open the workbook and define the worksheet
book = xlrd.open_workbook("Daily Flash.xlsx")
sheet = book.sheet_by_name("Sheet1")
query1 = """
CREATE TABLE [LEAF].[ZZZ] (
Lease_Number varchar(255),
Start_Date varchar(255),
Report_Status varchar(255),
Status_Date varchar(255),
Current_Status varchar(255),
Sales_Rep varchar(255),
Customer_Name varchar(255),
Total_Finance varchar(255),
Rate_Class varchar(255),
Supplier_Name varchar(255),
DecisionStatus varchar(255)
)"""
query = """
INSERT INTO [LEAF].[ZZZ] (
Lease_Number,
Start_Date,
Report_Status,
Status_Date,
Current_Status,
Sales_Rep,
Customer_Name,
Total_Finance,
Rate_Class,
Supplier_Name,
DecisionStatus
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"""
# execute create table
try:
cursor.execute(query1)
conn.commit()
except pyodbc.ProgrammingError:
pass
# grab existing row count in the database for validation later
cursor.execute("SELECT count(*) FROM LEAF.ZZZ")
before_import = cursor.fetchone()
for r in range(1, sheet.nrows):
Lease_Number = sheet.cell(r,0).value
Start_Date = sheet.cell(r,1).value
Report_Status = sheet.cell(r,2).value
Status_Date = sheet.cell(r,3).value
Current_Status= sheet.cell(r,4).value
Sales_Rep = sheet.cell(r,5).value
Customer_Name = sheet.cell(r,6).value
Total_Financed= sheet.cell(r,7).value
Rate_Class = sheet.cell(r,8).value
Supplier_Name = sheet.cell(r,9).value
DecisionStatus= sheet.cell(r,10).value
# Assign values from each row
values = (Lease_Number, Start_Date, Report_Status, Status_Date, Current_Status,
Sales_Rep, Customer_Name, Total_Financed, Rate_Class, Supplier_Name,
DecisionStatus)
# Execute sql Query
cursor.execute(query, values)
# Commit the transaction
conn.commit()
# If you want to check if all rows are imported
cursor.execute("SELECT count(*) FROM LEAF.ZZZ")
result = cursor.fetchone()
print((result[0] - before_import[0]) == len(data.index)) # should be True
# Close the database connection
conn.close()
I am new to Python and am trying to create a form that will save the data to a DB via PyMySQL, but I'm having an issue connecting to the DB. I have followed an example code (my version is below) and checked all syntax and speech marks thoroughly but am still getting the following error.
File "c:\Users\Nimrod\Nimrod Dropbox\Dropbox\Nimrodsky\Python\Adiabatic Equation\DB connect with PyMySQL.py", line 10, in <module> charset='utf8')
File "C:\ProgramData\Anaconda3\lib\site-packages\pymysql\__init__.py", line 90, in Connect return Connection(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\pymysql\connections.py", line 706, in __init__ self.connect()
File "C:\ProgramData\Anaconda3\lib\site-packages\pymysql\connections.py", line 922, in connect self.host_info = "socket %s:%d" % (self.host, self.port)
TypeError: %d format: a number is required, not str
import pymysql
import pymysql.cursors
# Connect to the database
connection = pymysql.connect(host='sql8.freesqldatabase.com',
port='3306',
user='********',
password='**********',
db='*********',
charset='utf8')
try:
with connection.cursor() as cursor:
# Create a new record
sql = "INSERT INTO `client` (`ID`, `client_name`, `client_address_1`, `client_address_2`, `client_address_3`, `client_postcode`, `occupier_name`, `install_address_1`, `install_address_2`, `install_address_3`, `install_postcode`) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
cursor.execute(sql, ('1', 'Terry Jones', '10 DOwning Street', 'Guildford','Surrey', 'GU1 5HA', 'Paul Smith', 'Wimbledon Hill', 'Wimbledon', 'London', 'SW19 5QT'))
#connection is not autocommit by default. So you must commit to save
#your changes.
connection.commit()
with connection.cursor() as cursor:
# Read a single record
sql = "SELECT `id`, `occupier_name` FROM `client` WHERE `client_name`=%s"
cursor.execute(sql, ('Terry Jones',))
result = cursor.fetchone()
print (result)
finally:
connection.close
Port must be an integer: port=3306.
I am learning python and i am new bie.
I am trying to use functions with mysql and python and i ma getting errors
This is my script
import MySQLdb
def insert_values(cursor, values):
#cursor = self.connection.cursor()
cursor.executemany("""
insert into pythontest (name1,name2,name3)
values (%s, %s, %s)""", values)
cursor.close()
db = MySQLdb.connect("localhost","root","root","python" )
cursor = db.cursor()
var1 = ['name1','name2','name3']
insert_values(cursor,var1)
db.close()
There may be many errors because i am learning
1)i don't know how can i pass db
object in function or passing cusrsor
is ok. because i have to call that
function many times in for loop
2)is the syntax of values array ok to
go in database
ERRORS
File "mysql.py", line 10, in insert_values
values (%s, %s, %s)""", values)
File "build/bdist.linux-i686/egg/MySQLdb/cursors.py", line 216, in executemany
File "build/bdist.linux-i686/egg/MySQLdb/connections.py", line 36, in defaulterrorhandler
_mysql_exceptions.ProgrammingError: not enough arguments for format string
cursor.executemany("""
insert into pythontest (name1,name2,name3)
values (%s, %s, %s)""", *values)
Here's how I would write that (But untested):
import MySQLdb
def insert_values(db, values):
cursor = db.cursor()
try:
try:
cursor.execute("""
insert into pythontest (name1,name2,name3)
values (%s, %s, %s)""", *values)
except:
db.rollback()
raise
else:
db.commit()
finally:
cursor.close()
db = MySQLdb.connect("localhost","root","root","python" )
vars = ('name1','name2','name3')
insert_values(db, vars)
db.close()
The cursor starts a transaction, so you don't want to re-use the same cursor for multiple updates unless they are part of an atomic transaction.