We Keep Coding

How to Capture just the value from a PostgreSQL Query

I am trying to capture the only the record from a PostgreSQL statement. The select statements outputs one row with column named as updated_at and the value is a timestamp- '2008-01-01 00:50:01'. I want to just capture/collect that value so when I call that variable, it just outputs '2008-01-01 00:50:01'.
Here is my code:
def get_etl_record():
pg_hook = PostgresHook(postgre_conn_id="post", schema='schema1')
connection = pg_hook.get_conn()
cursor = connection.cursor()
cursor2 = connection.cursor()
latest_update_query = "select max(updated_at) from my_table group by updated_at"
cursor.execute(latest_update_query)
#results= cursor.fetchall()
columns = [col[0] for col in cursor.description]
rows = [dict(zip(columns, row[0])) for row in cursor.fetchall()]
print(rows)
However this code doesnt give me an output.
Any ideas or suggestions?

There is no way to do what you want, but 3 ways to do very similar:
1.
cursor = connection.cursor()
cursor.execute(sql)
result = cursor.fetchone()
max_updated_at = result[0]
2.
dict_cur = connection.cursor(cursor_factory=psycopg2.extras.DictCursor)
dict_cur.execute('select max(updated_at) as max_updated_at ...')
result = dict_cur.fetchone()
max_updated_at = result['max_updated_at']
3.
nt_cur = connection.cursor(cursor_factory=psycopg2.extras.NamedTupleCursor)
nt_cur.execute('select max(updated_at) as max_updated_at ...')
result = nt_cur.fetchone()
max_updated_at = result.max_updated_at

Updating results from a mysql-connector fetchall

I'm trying to select certain records from the civicrm_address table and update the geocode columns. I use fetchall to retrieve the rows then, within the same loop, I try to update with the results of the geocoder API, passing the civicrm_address.id value in the update_sql statement.
The rowcount after the attempted update and commit is always -1 so I am assuming it failed for some reason but I have yet to figure out why.
import geocoder
import mysql.connector
mydb = mysql.connector.connect(
[redacted]
)
mycursor = mydb.cursor(dictionary=True)
update_cursor = mydb.cursor()
sql = """
select
a.id
, street_address
, city
, abbreviation
from
civicrm_address a
, civicrm_state_province b
where
location_type_id = 6
and
a.state_province_id = b.id
and
street_address is not null
and
city is not null
limit 5
"""
mycursor.execute(sql)
rows = mycursor.fetchall()
print(mycursor.rowcount, "records selected")
for row in rows:
address_id = int(row["id"])
street_address = str(row["street_address"])
city = str(row["city"])
state = str(row["abbreviation"])
myaddress = street_address + " " + city + ", " + state
g = geocoder.arcgis(myaddress)
d = g.json
latitude = d["lat"]
longitude = d["lng"]
update_sql = """
begin work;
update
civicrm_address
set
geo_code_1 = %s
, geo_code_2 = %s
where
id = %s
"""
var=(latitude, longitude, address_id)
print(var)
update_cursor.execute(update_sql, var, multi=True)
mydb.commit()
print(update_cursor.rowcount)
mycursor.close()
update_cursor.close()
mydb.close()
Here is a simpler script:
I have executed the update_sql statement directly in the MySQL workbench and it succeeds. It is not working from Python.
import geocoder
import mysql.connector
try:
mydb = mysql.connector.connect(
[redacted]
)
mycursor = mydb.cursor(dictionary=True)
update_cursor = mydb.cursor()
update_sql = """
begin work;
update
civicrm_address
set
geo_code_1 = 37.3445
, geo_code_2 = -118.5366074
where
id = 65450;
"""
update_cursor.execute(update_sql, multi=True)
mydb.commit()
print(update_cursor.rowcount, "row(s) were updated")
except mysql.connector.Error as error:
print("Failed to update record to database: {}".format(error))
mydb.rollback()
finally:
# closing database connection.
if (mydb.is_connected()):
mydb.close()

I have it working now. I did remove the "begin work" statement but not the multi=True and it wouldn't work. Later I removed the multi=True statement and it works.

python to write data into table error

write python program to create a mysql table and insert data into this table,the program is as follows:
def pre_data_db_manage(type,data):
conn = pymysql.connect(host="localhost", port=3306, user="root", passwd="********", db="facebook_info",charset="utf8")
cur = conn.cursor()
if type == "pre_davi_group_members_data":
is_exist_table_sql = "SHOW TABLES LIKE 'fb_pre_davi_group_members_posts'"
if cur.execute(is_exist_table_sql) == 0:
create_table_sql = '''CREATE TABLE fb_pre_davi_group_members_posts (id bigint not null primary key auto_increment,userID bigint,userName varchar(128),userURL varchar(256),
postTime varchar(128),postText text,postTextLength int,likesCount int,sharesCount int,commentsCount int,postTextPolarity varchar(64),postTextSubjectivity varchar(64))'''
cur.execute(create_table_sql)
r = re.compile(r'^[a-zA-Z0-9]')
for item in data:
if "'" in item["PostText"]:
item["PostText"] = item["PostText"].replace("'"," ")
if "\\" in item["PostText"]:
item["PostText"] = item["PostText"].replace("\\","\\\\")
for i in item["PostText"]:
result = r.match(i)
if result == None:
print("in re")
item['PostText'] = item['PostText'].replace(i, ' ')
if "nan" in item["SharesCount"]:
item["SharesCount"] = 0
if "nan" in item["LikesCount"]:
item["LikesCount"] = 0
if "nan" in item["CommentsCount"]:
item["CommentsCount"] = 0
if "nan" in item["PostTextLength"]:
item["PostTextLength"] = 0
item["PostTextLength"] = int(item["PostTextLength"])
item["LikesCount"] = int(item["LikesCount"])
item["SharesCount"] = int(item["SharesCount"])
item["CommentsCount"] = int(item["CommentsCount"])
if type == "pre_davi_group_members_data":
insert_sql = '''INSERT INTO fb_pre_davi_group_members_posts (userID,userName,userURL,
postTime,postText,postTextLength,likesCount,sharesCount,commentsCount,postTextPolarity,postTextSubjectivity) VALUES
({0},"{1}",'{2}','{3}','{4}',{5},{6},{7},{8},{9},{10})'''.format(item["UserID"],item["UserName"],item["UserURL"],item["PostTime"],item["PostText"],item["PostTextLength"],item["LikesCount"],item["SharesCount"],item["CommentsCount"],item["PostTextPolarity"],item["PostTextSubjectivity"])
print(insert_sql)
try:
cur.execute(insert_sql)
except Exception as e:
print("insert error")
continue
cur.close()
conn.commit()
conn.close()
and write call statement as follows:
type = "pre_davi_group_members_data"
pre_data_db_manage(type, df_list)
however,when execute this program, found that no data have been inserted into table:fb_pre_davi_group_members_posts,
in the mysql order line, write:
select count(*) from fb_pre_davi_group_members_posts;
the result is 0
could you please tell me the reason and how to solve it

PyMySql cursor fetch all return empty when database has rows in python

Following is the code, i get the result as empty array but cursor has rows shows rowcount > 1 :
``
def __init__(self,readLink):
if readLink==0:
self.linksToRead = 1000000000
else:
self.linksToRead = readLink
self.linkCount = 0
self.wordsList =[]
self.parsedLinks=[]
self.urlList =[]
self.connection = pymysql.connect(host='localhost',
user='root',
password='s3cr3tp#ssw0rd',
db='scrapper',
charset='utf8',
cursorclass=pymysql.cursors.DictCursor, autocommit=True)
with self.connection.cursor() as cursor:
sql2 = "SELECT * FROM `linksparsed`"
try:
cursor.execute(sql2)
#self.connection.commit()
except Exception as ex:
print(ex)
result = cursor.fetchall()
cursor.rowcount #shows 3
totalLength = len(result) # shows 0
for row in result:
self.parsedLinks.append(row)

What is the rownumber after fetchall() ?
if it is 3, equals to rowcount, means have already fetched rows.
This is the source codes of fetchall, suppose return at "result self._rows[self.rownumber:]"
def fetchall(self):
"""Fetch all the rows"""
self._check_executed()
if self._rows is None:
return ()
if self.rownumber:
result = self._rows[self.rownumber:] <<-------
else:
result = self._rows
self.rownumber = len(self._rows)
return result

python Sqlite3 parameter subs

I have a small problem with this class which handle my DB. It still saying:
cursor.execute(sql)
ValueError: operation parameter must be str
I tried lots of things but nothing work as i want. I looked over https://docs.python.org/3.4/library/sqlite3.html and i'm sure i do the same things.
import sqlite3
class Database():
def __init__(self):
try:
self.db = sqlite3.connect('../database.sqlite')
self.cur = self.db.cursor()
self.cur.execute('pragma foreign_keys="1"')
except sqlite3.Error as e:
raise e
def select(self,sql):
cursor = self.db.cursor()
cursor.execute(sql)
records = cursor.fetchall()
cursor.close()
return records
def insert(self,sql):
cursor = self.db.cursor()
cursor.execute(sql)
newID = cursor.lastrowid
self.db.commit()
cursor.close()
return newID
def execute(self,sql):
""" execute any SQL statement but no return value given """
cursor = self.db.cursor()
cursor.execute(sql)
self.db.commit()
cursor.close()
if __name__ == '__main__':
db = Database()
#sql = "SELECT skuref, titre_prod FROM product"
t = ("888888",)
sql= "UPDATE product SET created = 1 WHERE skuref = ?", t
db.execute(sql)
If someone can help me it would be grateful.Later i wanted to pass something like this in the main program inside a for loop
lastpost = record[0]
if created = True
sql = "UPDATE product SET created = 1 WHERE skuref = ?",(lastpost,)
db.execute(sql)

sql is a tuple containing SQL statement and the parameters.
Change as following, so that sql and parameters are passed separately, instead of being passed as a tuple:
def execute(self, sql):
""" execute any SQL statement but no return value given """
cursor = self.db.cursor()
cursor.execute(*sql) # <------
self.db.commit()
cursor.close()

With your statement
sql = "UPDATE product SET created = 1 WHERE skuref = ?",(lastpost,)
you have created a tupel like
("UPDATE product SET created = 1 WHERE skuref = ?", (lastpost,))
You have to give the arguments as parameters to the execute() function.
Also your if statement is bad: no :, = instead of == and the whole check for True is no nesesary.
Try this:
lastpost = record[0]
if created:
sql = "UPDATE product SET created = 1 WHERE skuref = ?"
db.execute(sql, lastpost)