We Keep Coding

Anyway to Upsert database using PostgreSQL in Python

I want to upsert with least effort, for simplicity, i reduce columns, this not work:
sql = '''INSERT INTO temp.tickets
(id, created_at, updated_at, emails, status)
VALUES
(%s, %s, %s, %s, %s)
ON CONFLICT (id)
DO UPDATE SET ( emails, status) values (%s,%s)
'''
cursor = cm.cursor()
## cm is a custom module
cursor.execute(sql, (ticket['id'],
ticket['created_at'],
ticket['updated_at'],
ticket['emails'], ticket['status'], )
This code show Error:
return super(DictCursor, self).execute(query, vars)
IndexError: tuple index out of range
What I need to change in the cursor.execute() to work?
The Bellow code work but I like to use %s instead of type: email = excluded.email for each columns
sql = '''INSERT INTO temp.tickets
(id, created_at, updated_at, emails, status)
VALUES
(%s, %s, %s, %s, %s)
ON CONFLICT (id)
DO UPDATE SET emails = excluded.eamils, status = excluded.status
'''
cursor = cm.cursor()
# cm is a custom module
cursor.execute(sql, (ticket['id'],
ticket['created_at'],
ticket['updated_at'],
ticket['emails'], ticket['status'], )
There are two Relevant Questions link1, link2

I would try something like this:
sql = '''INSERT INTO temp.tickets
(id, created_at, updated_at, emails, status)
VALUES
(%s, %s, %s, %s, %s)
ON CONFLICT (id)
DO UPDATE SET ( emails, status) values (%s,%s)
'''
cursor = cm.cursor()
## cm is a custom module
cursor.execute(sql, (ticket['id'],
ticket['created_at'],
ticket['updated_at'],
ticket['emails'],
ticket['status'],
ticket['emails'],
ticket['status'] )
Thre number of %s must match the number of parameters.

When Postgres encounters a captured conflict it basically creates a record called EXCLUDED that contains the values you attempted to insert, You can refer to this record in DO UPDATE. Try the following:
INSERT INTO temp.tickets
(id, created_at, updated_at, emails, status)
VALUES
(%s, %s, %s, %s, %s)
ON CONFLICT (id)
DO UPDATE
SET emails = excluded.emails
, status = excluded.status
, updated_at = excluded.updated_at -- my assumption.
...
You will have to format is into the requirements of your source language.

Error while updating MySQL DB from PostgreSQL DB

I need to update/insert rows to MySQL database using the data from Postgres DB.So here is the script which i'm using but getting the below error while i schedule this in Jenkins.
Can anyone please guide on what i can do/change to rectify this.
File "signup.py", line 80, in <module>
11:59:27 cur_msql_1.execute(msql_insert_1, row)
11:59:27 File "/usr/local/lib/python3.5/dist-packages/MySQLdb/cursors.py", line 209, in execute
11:59:27 res = self._query(query)
11:59:27 File "/usr/local/lib/python3.5/dist-packages/MySQLdb/cursors.py", line 315, in _query
11:59:27 db.query(q)
11:59:27 File "/usr/local/lib/python3.5/dist-packages/MySQLdb/connections.py", line 239, in query
11:59:27 _mysql.connection.query(self, query)
11:59:27 MySQLdb._exceptions.ProgrammingError: (1064, 'You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near \'"timestamp", ip, store_id, confirmed_at) SELECT \'user123#gmail.com\', 15463\' at line 2')
11:59:27 Build step 'Execute shell' marked build as failure
11:59:27 Finished: FAILURE
Below is the entire code:
import psycopg2
import os
import time
import MySQLdb
import sys
from pprint import pprint
from datetime import datetime
from utils.config import Configuration as Config
from utils.postgres_helper import get_connection
from utils.utils import get_global_config
# MySQLdb connection
try:
source_host = 'magento'
conf = get_global_config()
cnx_msql = MySQLdb.connect(host=conf.get(source_host, 'host'),
user=conf.get(source_host, 'user'),
passwd=conf.get(source_host, 'password'),
port=int(conf.get(source_host, 'port')),
db=conf.get(source_host, 'db'))
print('Magento MySQL DB Connected')
except mysql.connector.Error as e:
print ("MYSQL: Unable to connect!", e.msg)
sys.exit(1)
# Postgresql connection
try:
cnx_psql = get_connection(get_global_config(), 'pg_dwh')
print('DWH PostgreSQL DB Connected')
except psycopg2.Error as e:
print('PSQL: Unable to connect!\n{0}').format(e)
sys.exit(1)
# Cursors initializations
cur_msql = cnx_msql.cursor()
cur_msql_1 = cnx_msql.cursor()
cur_psql = cnx_psql.cursor()
cur_psql_1 = cnx_psql.cursor()
now = time.strftime('%Y-%m-%d %H:%M:%S')
##################################################################################
update_sql_base="""select gr.email from unsubscribed_contacts gr
INNER JOIN subscriber sn on sn.email=gr.email"""
msql_update_1="""UPDATE subscriber SET status=3,timestamp=CAST(TO_CHAR(now(),'YYYY-MM-DD HH24:MI:SS') AS TIMESTAMP) WHERE email='%s'"""
msql_update_2="""UPDATE n_subscriber SET subscriber_status=3,change_status_at=CAST(TO_CHAR(now(),'YYYY-MM-DD HH24:MI:SS') AS TIMESTAMP)
WHERE subscriber_email='%s';"""
cur_psql.execute(update_sql_base)
for row in cur_psql:
email=row[0]
cur_msql.execute(msql_update_1 %email)
cnx_msql.commit()
cur_msql.execute(msql_update_2 %email)
cnx_msql.commit()
##################################################################################
insert_sql_base="""select gr.email,c.customer_id,'',3,'',CAST(TO_CHAR(now(),'YYYY-MM-DD HH24:MI:SS') AS TIMESTAMP),'','',CAST(TO_CHAR(now(),'YYYY-MM-DD HH24:MI:SS') AS TIMESTAMP)
from unsubscribed_contacts gr
LEFT JOIN n_subscriber sn on sn.email=gr.email
LEFT JOIN customers_2 c on c.customer_email=gr.email
WHERE sn.email IS NULL"""
msql_insert="""INSERT INTO n_subscriber(
email, customer_id, options, status, confirm_code, "timestamp", ip, store_id, confirmed_at) SELECT """
msql_insert_1="""INSERT INTO n_subscriber(
email, customer_id, options, status, confirm_code, "timestamp", ip, store_id, confirmed_at) SELECT %s, %s, %s, %s, %s, %s, %s, %s, %s"""
cur_psql_1.execute(insert_sql_base)
for row in cur_psql_1:
print(msql_insert_1)
cur_msql_1.execute(msql_insert_1, row)
cnx_msql.commit()
## Closing cursors'
cur_msql.close()
cur_psql.close()
cur_psql_1.close()
cur_msql_1.close()
## Closing database connections
cnx_msql.close()
cnx_psql.close()
Python : 3.5
PostgreSQL: Version 11

The main problem is wrong syntax(cur_msql_1.execute(msql_insert_1, row)). Just trying to explain using a few tables:
create table subscriber
(
customer_id int null,
email varchar(100) null,
timestamp int null
);
INSERT INTO subscriber (customer_id, email, timestamp) VALUES (1, 'test1#gmail.com', 1591187277);
INSERT INTO subscriber (customer_id, email, timestamp) VALUES (2, 'test2#gmail.com', 1591187303);
create table n_subscriber
(
customer_id int null,
email varchar(100) null,
timestamp int null
);
in your case it works something like this:
import MySQLdb
db = MySQLdb.connect(...)
cursor = db.cursor()
cursor.execute("SELECT customer_id, email, timestamp FROM subscriber")
for row in cursor:
cursor.execute("""INSERT INTO n_subscriber(customer_id, email, "timestamp") SELECT %s, %s, %s""", row)
db.commit()
MySQLdb._exceptions.ProgrammingError: (1064, 'You have an error in
your SQL syntax; check the manual that corresponds to your MySQL
server version for the right syntax to use near \'"timestamp") SELECT
1, \'test1#gmail.com\', 1591187277\' at line 1')
Correct syntax:
cursor.execute("INSERT INTO n_subscriber(customer_id, email, timestamp) VALUES (%s, %s, %s)", row)
Also you can do it using executemany():
cursor = db.cursor()
cursor.execute("SELECT customer_id, email, timestamp FROM subscriber")
data = cursor.fetchall()
cursor.executemany("INSERT INTO n_subscriber(customer_id, email, timestamp) VALUES (%s, %s, %s)", data)
db.commit()
Hope this helps.

Python objects to JSON to MySQL

I've got a problem when trying to insert a json, which was converted from a python object with json.dumps, into a MySQL database. The connection to the database is working with another python file. I've already tried to just insert values, which was working, but with the json file it's not working.
My Python file:
import json
import dbConnection
cur = dbConnection.cursor
cnx = dbConnection.conn
DEVICES = {
"id": "1",
"isPoweredOn": "True",
"os": "Linux"
}
j = json.dumps(DEVICES)
print(j)
sql = "INSERT INTO DEVICES (id, isPoweredOn, os) VALUES (%s, %s, %s)"
val = (json.dumps(DEVICES))
cur.execute(sql, val)
cnx.commit()
print(cur.rowcount, "record inserted.")
Error code I get, when trying to execute:
"id": "1", "isPoweredOn": "True", "os": "Linux"}
Traceback (most recent call last):
File "dbInit.py", line 22, in <module>
cur.execute(sql, val)
File "/home/silvan/.virtualenvs/pyproj1/lib/python3.8/site-packages/mysql/connector/cursor.py", line 551, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "/home/silvan/.virtualenvs/pyproj1/lib/python3.8/site-packages/mysql/connector/connection.py", line 490, in cmd_query
result = self._handle_result(self._send_cmd(ServerCmd.QUERY, query))
File "/home/silvan/.virtualenvs/pyproj1/lib/python3.8/site-packages/mysql/connector/connection.py", line 395, in _handle_result
raise errors.get_exception(packet)
mysql.connector.errors.ProgrammingError: 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '%s, %s, %s)' at line 1
My CREATE TABLE code:
CREATE TABLE DEVICES(id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, isPoweredOn BOOLEAN NOT NULL, os VARCHAR(50) NOT NULL);
Thanks for any help in advance!

You need to json.loads(j) and assign it to a variable, then you can access the values properly.
Try :
import json
import dbConnection
cur = dbConnection.cursor
cnx = dbConnection.conn
DEVICES = {
"id": "1",
"isPoweredOn": False ,
"os": "Linux"
}
j = json.dumps(DEVICES)
values = json.loads(j)
'''
# Quick debugging
print(j , type(j))
print(values , type(values))
print(values['isPoweredOn'])
'''
sql = "INSERT INTO DEVICES (id, isPoweredOn, os) VALUES (%s, %s, %s)"
val = ( '' , values['isPoweredOn'] , values['os'])
cur.execute(sql, val)
cnx.commit()
print(cur.rowcount, "record inserted.")
Also since you defined id to be INT AUTO_INCREMENT PRIMARY KEY NOT NULL it , you can't insert device id wich is values['id'] to id column, you can alter DEVICES table and create a new column called device_id for storing the device id if you need really need to store values['id']

Firstly, cast the DEVICES to dict, then Here's the format.
sql = "INSERT INTO DEVICES (`id`, `isPoweredOn`, `os`) VALUES (%(id)s, %(isPoweredOn)s, %(os)s)"
Then Execute it :
try:
cur.execute(sql, DEVICES)
cnx.commit()
except error:
print(error)
Cheers!!

Why does execute () append null values and executemany() doesn't?

While using MySQL through python, on using execute() function null values are getting updated in the specific table, but on using executemany() the null values are returning errors.
The working commands:
mycursor = mydb.cursor()
mycursor.execute("INSERT INTO users (name, fav) VALUES('John', null)"
mydb.commit()
The code that doesn't work:
mycursor = mydb.cursor()
sql = "INSERT INTO users (name, fav) VALUES(%s, %s)"
val =[
('Samantha', 154),
('Thalia', 155),
('Jacobs', null),
('Jamie', null)
]
mycursor.executemany(sql, val)
mydb.commit()
The error generated is as follows:
Traceback (most recent call last):
File "demo_python.py", line 14, in
('Jacobs', null),
NameError: name 'null' is not defined

MySQLdb: Operand should contain 1 column(s)

I am trying to insert some data into a MySQL database, using Python and MySQLdb. When I execute the following function in my program, MySQL returns error "1241, 'Operand should contain 1 column(s)'"
User, password and database are correct, the table is existing and all rights are granted.
def write_to_mysql(pname, dat, like, reachs, talker, friendsfans):
''
try:
con = mdb.connect(user='user', passwd='password', host='localhost', db='database');
except Exception. err:
print(err)
with con:
cur = con.cursor()
cur.execute("INSERT INTO fbinsights (page, datum, likes, reach, talking, fanfriends) VALUES( %s, %s, %s, %s, %s, %s)", (pname, dat, like, reachs, talker, friendsfans))
connection.commit()
Where's the mistake?
Full traceback:
File "insights.py", line 111, in <module>
main()
File "insights.py", line 108, in main
write_to_mysql(PAGE_NAME, date, likes_atm, reach_day, talking_day, friends_of_fans)
File "insights.py", line 90, in write_to_mysql
cur.execute("INSERT INTO fbinsights (page, datum, likes, reach, talking, fanfriends) VALUES( %s, %s, %s, %s, %s, %s)", (pname, dat, like, reachs, talker, friendsfans))
File "/usr/local/lib/python2.7/site-packages/MySQL_python-1.2.3-py2.7-freebsd-9.0-RELEASE-p3-amd64.egg/MySQLdb/cursors.py", line 174, in execute
self.errorhandler(self, exc, value)
File "/usr/local/lib/python2.7/site-packages/MySQL_python-1.2.3-py2.7-freebsd-9.0-RELEASE-p3-amd64.egg/MySQLdb/connections.py", line 36, in defaulterrorhandler
raise errorclass, errorvalue
_mysql_exceptions.OperationalError: (1241, 'Operand should contain 1 column(s)')

#schlamar answered it. Wrong types passed to MySQL.

I had this error when I was generating a SELECT query with columns to select enclosed (by mistake) in parentheses: SELECT (id, name, age) FROM members;
Note that it does not raise this error if you have just one column listed in parentheses.