We Keep Coding

Error while updating MySQL DB from PostgreSQL DB

I need to update/insert rows to MySQL database using the data from Postgres DB.So here is the script which i'm using but getting the below error while i schedule this in Jenkins.
Can anyone please guide on what i can do/change to rectify this.
File "signup.py", line 80, in <module>
11:59:27 cur_msql_1.execute(msql_insert_1, row)
11:59:27 File "/usr/local/lib/python3.5/dist-packages/MySQLdb/cursors.py", line 209, in execute
11:59:27 res = self._query(query)
11:59:27 File "/usr/local/lib/python3.5/dist-packages/MySQLdb/cursors.py", line 315, in _query
11:59:27 db.query(q)
11:59:27 File "/usr/local/lib/python3.5/dist-packages/MySQLdb/connections.py", line 239, in query
11:59:27 _mysql.connection.query(self, query)
11:59:27 MySQLdb._exceptions.ProgrammingError: (1064, 'You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near \'"timestamp", ip, store_id, confirmed_at) SELECT \'user123#gmail.com\', 15463\' at line 2')
11:59:27 Build step 'Execute shell' marked build as failure
11:59:27 Finished: FAILURE
Below is the entire code:
import psycopg2
import os
import time
import MySQLdb
import sys
from pprint import pprint
from datetime import datetime
from utils.config import Configuration as Config
from utils.postgres_helper import get_connection
from utils.utils import get_global_config
# MySQLdb connection
try:
source_host = 'magento'
conf = get_global_config()
cnx_msql = MySQLdb.connect(host=conf.get(source_host, 'host'),
user=conf.get(source_host, 'user'),
passwd=conf.get(source_host, 'password'),
port=int(conf.get(source_host, 'port')),
db=conf.get(source_host, 'db'))
print('Magento MySQL DB Connected')
except mysql.connector.Error as e:
print ("MYSQL: Unable to connect!", e.msg)
sys.exit(1)
# Postgresql connection
try:
cnx_psql = get_connection(get_global_config(), 'pg_dwh')
print('DWH PostgreSQL DB Connected')
except psycopg2.Error as e:
print('PSQL: Unable to connect!\n{0}').format(e)
sys.exit(1)
# Cursors initializations
cur_msql = cnx_msql.cursor()
cur_msql_1 = cnx_msql.cursor()
cur_psql = cnx_psql.cursor()
cur_psql_1 = cnx_psql.cursor()
now = time.strftime('%Y-%m-%d %H:%M:%S')
##################################################################################
update_sql_base="""select gr.email from unsubscribed_contacts gr
INNER JOIN subscriber sn on sn.email=gr.email"""
msql_update_1="""UPDATE subscriber SET status=3,timestamp=CAST(TO_CHAR(now(),'YYYY-MM-DD HH24:MI:SS') AS TIMESTAMP) WHERE email='%s'"""
msql_update_2="""UPDATE n_subscriber SET subscriber_status=3,change_status_at=CAST(TO_CHAR(now(),'YYYY-MM-DD HH24:MI:SS') AS TIMESTAMP)
WHERE subscriber_email='%s';"""
cur_psql.execute(update_sql_base)
for row in cur_psql:
email=row[0]
cur_msql.execute(msql_update_1 %email)
cnx_msql.commit()
cur_msql.execute(msql_update_2 %email)
cnx_msql.commit()
##################################################################################
insert_sql_base="""select gr.email,c.customer_id,'',3,'',CAST(TO_CHAR(now(),'YYYY-MM-DD HH24:MI:SS') AS TIMESTAMP),'','',CAST(TO_CHAR(now(),'YYYY-MM-DD HH24:MI:SS') AS TIMESTAMP)
from unsubscribed_contacts gr
LEFT JOIN n_subscriber sn on sn.email=gr.email
LEFT JOIN customers_2 c on c.customer_email=gr.email
WHERE sn.email IS NULL"""
msql_insert="""INSERT INTO n_subscriber(
email, customer_id, options, status, confirm_code, "timestamp", ip, store_id, confirmed_at) SELECT """
msql_insert_1="""INSERT INTO n_subscriber(
email, customer_id, options, status, confirm_code, "timestamp", ip, store_id, confirmed_at) SELECT %s, %s, %s, %s, %s, %s, %s, %s, %s"""
cur_psql_1.execute(insert_sql_base)
for row in cur_psql_1:
print(msql_insert_1)
cur_msql_1.execute(msql_insert_1, row)
cnx_msql.commit()
## Closing cursors'
cur_msql.close()
cur_psql.close()
cur_psql_1.close()
cur_msql_1.close()
## Closing database connections
cnx_msql.close()
cnx_psql.close()
Python : 3.5
PostgreSQL: Version 11

The main problem is wrong syntax(cur_msql_1.execute(msql_insert_1, row)). Just trying to explain using a few tables:
create table subscriber
(
customer_id int null,
email varchar(100) null,
timestamp int null
);
INSERT INTO subscriber (customer_id, email, timestamp) VALUES (1, 'test1#gmail.com', 1591187277);
INSERT INTO subscriber (customer_id, email, timestamp) VALUES (2, 'test2#gmail.com', 1591187303);
create table n_subscriber
(
customer_id int null,
email varchar(100) null,
timestamp int null
);
in your case it works something like this:
import MySQLdb
db = MySQLdb.connect(...)
cursor = db.cursor()
cursor.execute("SELECT customer_id, email, timestamp FROM subscriber")
for row in cursor:
cursor.execute("""INSERT INTO n_subscriber(customer_id, email, "timestamp") SELECT %s, %s, %s""", row)
db.commit()
MySQLdb._exceptions.ProgrammingError: (1064, 'You have an error in
your SQL syntax; check the manual that corresponds to your MySQL
server version for the right syntax to use near \'"timestamp") SELECT
1, \'test1#gmail.com\', 1591187277\' at line 1')
Correct syntax:
cursor.execute("INSERT INTO n_subscriber(customer_id, email, timestamp) VALUES (%s, %s, %s)", row)
Also you can do it using executemany():
cursor = db.cursor()
cursor.execute("SELECT customer_id, email, timestamp FROM subscriber")
data = cursor.fetchall()
cursor.executemany("INSERT INTO n_subscriber(customer_id, email, timestamp) VALUES (%s, %s, %s)", data)
db.commit()
Hope this helps.

Why does execute () append null values and executemany() doesn't?

While using MySQL through python, on using execute() function null values are getting updated in the specific table, but on using executemany() the null values are returning errors.
The working commands:
mycursor = mydb.cursor()
mycursor.execute("INSERT INTO users (name, fav) VALUES('John', null)"
mydb.commit()
The code that doesn't work:
mycursor = mydb.cursor()
sql = "INSERT INTO users (name, fav) VALUES(%s, %s)"
val =[
('Samantha', 154),
('Thalia', 155),
('Jacobs', null),
('Jamie', null)
]
mycursor.executemany(sql, val)
mydb.commit()
The error generated is as follows:
Traceback (most recent call last):
File "demo_python.py", line 14, in
('Jacobs', null),
NameError: name 'null' is not defined

psycopg2.ProgrammingError: column of relation does not exist

Trying to insert data into PostgreSQL database.
Python code:
myFields = ((DOT_Number,),(Entity_Type,),(Operating_Status,),(Legal_Name,),
(Phone,),(Address,)
)
query = """ INSERT INTO saferdb_question( DOT_Number, Entity_Type, Operating_Status, Legal_Name, Phone, Address)VALUES ( %s, %s, %s, %s, %s, %s);"""
cur.execute( query, myFields)
Getting error:
Traceback (most recent call last):
File "scraper.py", line 189, in <module>
cur.execute( query, myFields)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/psycopg2/extras.py", line 144, in execute
return super(DictCursor, self).execute(query, vars)
psycopg2.ProgrammingError: column "dot_number" of relation "saferdb_question" does not exist
LINE 1: INSERT INTO saferdb_question( DOT_Number, Entity_Type, Oper...
SQL from PostgreSQL that created the table:
CREATE TABLE public.saferdb_question
(
id integer NOT NULL DEFAULT nextval('saferdb_question_id_seq'::regclass),
"DOT_Number" character varying(10) COLLATE pg_catalog."default" NOT NULL,
...
"Phone" character varying(15) COLLATE pg_catalog."default" NOT NULL,
"Address" character varying(200) COLLATE pg_catalog."default" NOT NULL,
)
WITH (
OIDS = FALSE
)
TABLESPACE pg_default;
ALTER TABLE public.saferdb_question
OWNER to postgres;

Finally solved this.
The columns had to be in quotes. Here is the final Query string:
query = """ INSERT INTO public.saferdb_question( "DOT_Number", "Entity_Type", "Operating_Status", "Legal_Name", "Phone", "Address")VALUES ( %s, %s, %s, %s, %s, %s);"""

MariaDB executemany on duplicate key in Debian Stretch

Could anyone explain me is it bug or feature?
Debian Stretch
mariadb-server-10.1.26
mariadb-client-10.1.26
MySQLdb-1.2.5
This python code perfectlly works in Debian Jessie, but failed in Stretch with error:
Traceback (most recent call last):
File "bug_check.py", line 17, in <module>
cur.executemany(q, p)
File "/usr/local/lib/python2.7/dist-packages/MySQLdb/cursors.py", line 255, in executemany
self.errorhandler(self, TypeError, msg)
File "/usr/local/lib/python2.7/dist-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler
raise errorclass, errorvalue
TypeError: not all arguments converted during string formatting
Python code:
#!/usr/bin/python
# -*- coding: UTF-8 *
import MySQLdb
db = MySQLdb.connect(host='192.168.1.183', user='root', passwd='password', db='test', charset='utf8')
cur = db.cursor()
q = """INSERT INTO test2 (id, value)
VALUES (%s, %s)
ON DUPLICATE KEY
UPDATE value=%s
"""
p = [(1, 7, 7)]
# failed
cur.executemany(q, p)
# working
for i in p:
cur.execute(q, i)
db.commit()
db.close()
Database:
CREATE TABLE `test2` (
`id` bigint(8) NOT NULL,
`value` float NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
ALTER TABLE `test2`
ADD PRIMARY KEY (`id`);
ALTER TABLE `test2`
MODIFY `id` bigint(8) NOT NULL AUTO_INCREMENT, AUTO_INCREMENT=1;
I am trying mariadb-10.2, pymysql but anyway error occurs in Stretch.

This may work: Change UPDATE value=%s to UPDATE value=VALUES(value) and get rid of the last 7 in the array.
If that does not work, then here is more discussion:
I think executemany is trying to build
INSERT ...
VALUES (...),
(...),
(...);
But it does not know how to convert the IODKU syntax into a repeated list like that. Bottom line: you can probably use executemany with INSERT, INSERT IGNORE, REPLACE, but not IODKU.
For IODKU to work, Stretch needs to be smart enough to do this:
INSERT INTO test2 (id, value)
VALUES
(%s, %s),
(%s, %s),
(%s, %s),
(%s, %s),
etc
ON DUPLICATE KEY
UPDATE value=VALUES(value)
Note that the repetition is in the middle, not on the end, as in the other cases. However you have to use the VALUES() pseudo-function to avoid the %s in the UPDATE clause.

Python "commands out of sync" error message when I am trying to import csv to MYSQLdb

I've get this error message when i want to execute my sqldb code in Python 2.7:
Traceback (most recent call last):
File "C:/Python27/air18-mysql.py", line 52, in <module>
cursor.execute(query)
File "C:\Python27\lib\site-packages\MySQLdb\cursors.py", line 221, in execute
if not self._defer_warnings: self._warning_check()
File "C:\Python27\lib\site-packages\MySQLdb\cursors.py", line 107, in _warning_check
warnings = self._get_db().show_warnings()
File "C:\Python27\lib\site-packages\MySQLdb\connections.py", line 371, in show_warnings
self.query("SHOW WARNINGS")
File "C:\Python27\lib\site-packages\MySQLdb\connections.py", line 282, in query
_mysql.connection.query(self, query)
ProgrammingError: (2014, "Commands out of sync; you can't run this command now")
The relevant pieces of my code are as follows:
connection = MySQLdb.connect(host='localhost',
user='root',
passwd='1234',
db='database')
cursor = connection.cursor()
query = """
CREATE TABLE `test` (
`A` varchar(100) DEFAULT NULL,
`B` varchar(100) DEFAULT NULL,
`C` varchar(100) DEFAULT NULL,
`D` varchar(100) DEFAULT NULL,
`E` varchar(100) DEFAULT NULL,
`F` varchar(100) DEFAULT NULL,
`G` varchar(100) DEFAULT NULL,
`ID` int(10) NOT NULL AUTO_INCREMENT,
PRIMARY KEY (`ID`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8"""
cursor.execute(query)
connection.commit()
cursor.close()
connection = MySQLdb.connect(host='localhost',
user='root',
passwd='1234',
db='database')
cursor = connection.cursor()
query = """ load data local infile 'C:/Python27/output.csv'
into table test
character set latin1
fields terminated by ';'
enclosed by '"'
lines terminated by '\r\n';
ignore 1 lines;
"""
cursor.execute(query)
connection.commit()
cursor.close()
Anybody know what I'm doing wrong?
I tried it with another csv too and it worked.
Maybe there is a problem in my csv file or i can't imagine what is the problem with this.
Here one part of my csv file (in NotePad++):
"apple apricot";" avocado";"blackcurrant (fruit)-";"blackberry";"blueberry (fruit) ";"";"lemon lime"
"quince pear";" banana";"papaya (fruit)-";"orange";"passion fruit (fruit) ";"";"pineapple watermelon"

after lemon lime and pineapple watermelon, you are missing a semi colon. Try with this:
"apple apricot";" avocado";"blackcurrant (fruit)-";"blackberry";"blueberry (fruit) ";"";"lemon lime";"quince pear";" banana";"papaya (fruit)";"orange";"passion fruit (fruit) ";"";"pineapple watermelon";