We Keep Coding

How to detect with feedparser if there are new items in an RSS channel?

I have the following code. When you understand the code, you can look at the two comments with
the capital letters. I could test if there are new items in the channel with insert or ignore but
I'm trying the better mechanism with utilization feed.updated_parsed attribute. Why doesn't it work
as expected?
from __future__ import unicode_literals
import feedparser
from sqlite3 import dbapi2 as sqlite
import sys, os
from datetime import datetime
from time import mktime
from daeutils import *
import re
import random
import optparse
import curses
import socket
def getActiveChannels():
"""Returns a list of active RSS channels"""
con = sqlite.connect(connectionString)
cur = con.cursor()
cur.execute("select id, title, xmlurl, updated from channels")
channels = cur.fetchall()
cur.close()
con.close()
return channels
def getItemsForChannel(xmlUrl, lastUpdate):
socket.setdefaulttimeout(60)
feedparserDictionary = feedparser.parse(xmlUrl)
updatedTime = datetime.fromtimestamp(mktime(feedparserDictionary.feed.updated_parsed))
lst = datetime.strptime(lastUpdate, "%Y-%m-%dT%H:%M:%S.%f")
if updatedTime < lst:
return [] # HERE NOT BEHAVING CORRECTLY, WHEN I COMMENT THIS LINE, THERE MAY BE A FEW ITEMS
items = feedparserDictionary.entries
print "There are new %d items" % len(items)
return items
def setChannelUpdateTime(xmlUrl, tm):
con = sqlite.connect(connectionString)
cur = con.cursor()
cur.execute("update channels set updated = :tm where xmlurl = :xmlUrl", locals())
con.commit()
print "updated successfully"
cur.close()
con.close()
if __name__ == "_main__":
con = sqlite.connect(connectionString)
for channel in getActiveChannels():
channelId, channelTitle, channelXmlUrl, lastChannelUpdate = channel
countOfNewItems = 0
items = getItemsForChannel(channelXmlUrl, lastChannelUpdate)
for item in items:
title, link, description, priority, updated = item
cur = con.cursor()
cur.execute("insert or ignore into feeds \
(title, link, description, read, updated, channelid) \
values (?, ?, ?, ?, ?, ?)", \
(title, link, description, 0, updated, channelId))
countOfNewItems += cur.rowcount # WHICH ARE INSERTED HERE
con.commit()
cur.close()
if countOfNewItems:
print "Found new items"
now = datetime.now().isoformat()
if "." not in now:
now = now + ".000000"
setChannelUpdateTime(channelXmlUrl, now)
Here are the two tables in sqlite:
CREATE TABLE channels (id integer primary key, title string, text string, description string, type string, xmlurl string unique, htmlurl string, priority integer, active integer, deactivated integer, updated text);
CREATE TABLE feeds (id integer primary key, title string, link string unique, description string, read integer, priority integer, updated string, channelid integer, foreign key (channelid) references channels(id));

I think the possible error is that you are trying to compare updated field on the feed, the feeds could be not well supported by the feed creator. Or timezone formatting because of using isoformat or etc.
Anyway, I believe that it is much better to compare PER ENTRY updated properties rather than comparing the feed property which is mostly used for invalidating feed cache.
Here is a working example, where I return only new entries from the function.
import socket
from datetime import datetime, timedelta
from time import mktime
import feedparser
from pprint import pprint
def getItemsForChannel(xmlUrl, lastUpdate):
lst = datetime.fromisoformat(lastUpdate)
socket.setdefaulttimeout(60)
parsed = feedparser.parse(xmlUrl)
items = [entry for entry in parsed.entries if
datetime.fromtimestamp(mktime(entry.updated_parsed)) > lst]
print("There are new {} items".format(len(items)))
return items
pprint(getItemsForChannel(
'http://serverfault.com/feeds/tag/+or+linux+or+ubuntu+or+vim+or+rsync+or+gnome',
(datetime.now() - timedelta(hours=3)).isoformat()
))
It uses from/to iso formatting for the last parsed date in your database value and compares entries per entry instead of global comparison based on the feed updated property.

Python SQLite3 insert return no error but no data in the table

I need your advice and help.
I am writing a code to parse region's names and the link of the corresponding region from a certain website. After that I want to store the region's names and the links in the database (sqlite3). The database was created, the table was created, however, the data could not be inserted to the table. I have tried some trials and errors but none worked. Therefore, I made this thread.
Here is my the code:
'''
usage python capstonePy.py http://www.liverpoolfc.com/fans/lfc-official-supporters-clubs
URL: http://www.liverpoolfc.com/fans/lfc-official-supporters-clubs
Official supporters URL pattern:
http://www.liverpoolfc.com/fans/lfc-official-supporters-clubs/[region]
'''
from sys import argv
from os.path import exists
from BeautifulSoup import *
import urllib
import re
import sqlite3
class FindSupporters:
def __init__(self, *args, **kwargs):
#parsing the url from the command line
url = argv[1]
#make a new database
cur = new_db('liverpudlian.sqlite3')
#open and read the url
fhand = open_and_read(url)
#print how many characters have been retrieved
suc_ret(len(fhand))
#make a list of links (href)
linklst = find_link(fhand)
#make a list of supporters regions
offsuplinklst = fans_link(linklst)
#make a new table and insert the data
officialsup_table(cur, offsuplinklst, 'liverpudlian.sqlite3')
sqlite3.connect('liverpudlian.sqlite3').close()
def new_db(name):
conn = sqlite3.connect(name)
cur = conn.cursor()
return cur
def open_and_read(url):
try:
fhand = urllib.urlopen(url).read()
except:
print '\n'
print "+------------------------------------------------------------------------------+"
print "|\t\t\t\tError: URL not found.\t\t\t\t|"
print "+------------------------------------------------------------------------------+"
print '\n'
quit()
return fhand
def suc_ret(length):
print '\n'
print "+------------------------------------------------------------------------------+"
print "|\t\t", length, "characters have been successfully retrieved\t\t|"
print "+------------------------------------------------------------------------------+"
print '\n'
def find_link(fhand):
links = []
tags = []
soup = BeautifulSoup(fhand)
tags = soup('a')
for tag in tags:
tag = tag.get('href',None)
if tag is not None :
links.append(tag)
return links
def fans_link(linklst):
offsuplinklst = []
for link in linklst:
link = str(link)
link = link.rstrip()
fans = re.findall('.*fans/.+clubs/(.+)', link)
if len(fans) > 0:
offsuplinklst.append(fans[0])
return offsuplinklst
def officialsup_table(cur, offsuplinklst, name):
cur.execute('''
create table if not exists OfficialSup
(ID integer primary key,
Region text unique,
Link text unique,
Retrieved integer)''')
cur.execute('select Region from OfficialSup where Retrieved = 1 limit 1')
try :
cur.fetchone()[0]'
except :
for i in range(len(offsuplinklst)):
reg = offsuplinklst[i]
link = 'http://www.liverpoolfc.com/fans/lfc-official-supporters-clubs/'+offsuplinklst[i]
cur.execute('insert into OfficialSup (Region, Link, Retrieved) values (?, ?, 1)', (reg, link))
sqlite3.connect(name).commit()
FindSupporters()
Probably the error in the officialsup_table method. Nevertheless, my attempts did not return any good results.
Thanks a lot!
Regards,
Arnold A.

You need to commit using the same connection instance that your cursor was created in. Improve new_db to return both conn and cur:
def new_db(name):
conn = sqlite3.connect(name)
cur = conn.cursor()
return conn, cur
You would need to read the results of the function differently now:
class FindSupporters:
def __init__(self, *args, **kwargs):
#parsing the url from the command line
url = argv[1]
#make a new database
conn, cur = new_db('liverpudlian.sqlite3')
# ...
Pass the connection object to the officialsup_table function as well and call commit():
def officialsup_table(conn, cur, offsuplinklst, name):
cur.execute('''
create table if not exists OfficialSup
(ID integer primary key,
Region text unique,
Link text unique,
Retrieved integer)''')
conn.commit()
cur.execute('select Region from OfficialSup where Retrieved = 1 limit 1')
try :
cur.fetchone()[0]
except :
for i in range(len(offsuplinklst)):
reg = offsuplinklst[i]
link = 'http://www.liverpoolfc.com/fans/lfc-official-supporters-clubs/'+offsuplinklst[i]
cur.execute('insert into OfficialSup (Region, Link, Retrieved) values (?, ?, 1)', (reg, link))
conn.commit()

Exporting specific MySQL data from table to Excel using Python

I'm fairly new to mysql-connector-python. I'm in the middle of learning how to use python to make my job a little easier. Could you please help?
I have a database with one table.Lets call it ‘table1’ The table is broken down with columns like this:
sent_time | delivered_time |id1_active |id2_active |id3_active |id1_inactive |id2_inactive |id3_inactive |location_active |location_inactive …..`lots more
Lets say that these are two or more customers delivering goods to and from each other. Each customer has three id#s.
Right now I want to see if I can at least print out id1_active and id2_active for all of them…ideally, I want to print the entire rows with a specific id mentioned in any of the columns…can you look below and see what I’m doing wrong and maybe help me figure out a solution for my ideal goal?
I keep getting this error
line 18, in
c.writerow([id1_active,id2_active] )
TypeError: ‘str’ does not support the buffer interface
This is what I have thus far…
I created a ‘config.ini’ file to make my life a bit easier
[mysql]
host = localhost
database = db_name
user = root
password = blahblah
I created a ‘python_mysql_dbconfig.py’
from configparser import ConfigParser
def read_db_config(filename=’config.ini’, section=’mysql’):
“”” Read database configuration file and return a dictionary object
:param filename: name of the configuration file
:param section: section of database configuration
:return: a dictionary of database parameters
“””
# create parser and read ini configuration file
parser = ConfigParser()
parser.read(filename)
# get section, default to mysql
db = {}
if parser.has_section(section):
items = parser.items(section)
for item in items:
db[item[0]] = item[1]
else:
raise Exception(‘{0} not found in the {1} file’.format(section, filename))
return db
Then I created ‘export.py’
from mysql.connector import MySQLConnection, Error
from python_mysql_dbconfig import read_db_config
import csv
filename=open(‘test.csv’,’wb’)
c=csv.writer(filename)
db_config = read_db_config()
conn = MySQLConnection(**db_config)
cursor = conn.cursor()
query = (“SELECT id1_active, id2_active from table1”)
cursor.execute(query)
for id1_active, id2_active in cursor:
c.writerow([id1_active,id2_active] )
cursor.close()
filename.close()
cnn.close()
Could you tell me what I’m doing wrong?

You need to fetch the data and loop through data instead of cursor:
from mysql.connector import MySQLConnection, Error
from python_mysql_dbconfig import read_db_config
import csv
filename=open(‘test.csv’,’wb’)
c=csv.writer(filename)
db_config = read_db_config()
conn = MySQLConnection(**db_config)
cursor = conn.cursor()
query = (“SELECT id1_active, id2_active from table1”)
cursor.execute(query)
#You first need to fetch data
data = cursor.fetchall()
for item in data:
c.writerow(item )
cursor.close()
filename.close()
cnn.close()

I started figuring things out using xlswriter. I was able to export all data from the table but now I cannot get my code to pull the data that falls between two datetimes....it keeps telling me
Traceback (most recent call last):
File "C:\Python34\timerange.py", line 36, in <module>
start_date = datetime.datetime(userIn,timeShape)
AttributeError: type object 'datetime.datetime' has no attribute 'datetime'
Please, can you tell me what I'm doing wrong?
# Establish a MySQL connection
from mysql.connector import MySQLConnection, Error
from python_mysql_dbconfig import read_db_config
db_config = read_db_config()
conn = MySQLConnection(**db_config)
import xlsxwriter
from xlsxwriter.workbook import Workbook
import datetime
from datetime import datetime
cursor = conn.cursor()
#creates the workbook
workbook = xlsxwriter.Workbook('imhere.xlsx')
worksheet = workbook.add_worksheet()
#formatting definitions
bold = workbook.add_format({'bold': True})
date_format = workbook.add_format({'num_format': 'yyyy-mm-dd hh:mm:ss'})
timeShape = '%Y-%m-%d %H:%M:%S'
query = ("SELECT sent_time, delivered_time, customer_name, id1_active, id2_active, id3_active, id1_inactive, id2_inactive, id3_inactive, location_active, location_inactive FROM table1 ")
"WHERE sent_time BETWEEN %s AND %s"
userIn = input("Type Start Date (YYYY-MM-DD hh:mm:ss):")
userEnd = input("Type End Date (YYYY-MM-DD hh:mm:ss):")
start_date = datetime.datetime(userIn,timeShape)
end_date = datetime.datetime(userEnd, timeShape)
# Execute sql Query
cursor.execute(query, (start_date, end_date))
#result = cursor.fetchall()
#sets up the header row
worksheet.write('A1','sent_time',bold)
worksheet.write('B1', 'delivered_time',bold)
worksheet.write('C1', 'customer_name',bold)
worksheet.write('D1', 'id1_active',bold)
worksheet.write('E1', 'id2_active',bold)
worksheet.write('F1', 'id3_active',bold)
worksheet.write('G1', 'id1_inactive',bold)
worksheet.write('H1', 'id2_inactive',bold)
worksheet.write('I1', 'id3_inactive',bold)
worksheet.write('J1', 'location_active',bold)
worksheet.write('K1', 'location_inactive',bold)
worksheet.autofilter('A1:K1')
print("sent_time", "delivered_time", "customer_name", "id1_active", "id2_active", "id3_active", "id1_inactive", "id2_inactive", "id3_inactive", "location_active", "location_inactive")
for row in cursor:
print(row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8], row[9],row[10])
# Create a For loop to iterate through each row in the XLS file, starting at row 2 to skip the headers
for r, row in enumerate(cursor, start=1): #where you want to start printing results inside workbook
for c, col in enumerate(row):
worksheet.write_datetime(r,0,row[0], date_format)
worksheet.write_datetime(r,1, row[1], date_format)
worksheet.write(r,2, row[2])
worksheet.write(r,3, row[3])
worksheet.write(r,4, row[4])
worksheet.write(r,5, row[5])
worksheet.write(r,6, row[6])
worksheet.write(r,7, row[7])
worksheet.write(r,8, row[8])
worksheet.write(r,9, row[9])
worksheet.write(r,10, row[10])
#close out everything and save
cursor.close()
workbook.close()
conn.close()
#print number of rows and bye-bye message
print ("- - - - - - - - - - - - -")
rows = len(result)
print ("I just imported "+ str(rows) + " rows from MySQL!")
print ("")
print ("Good to Go!!!")
print ("")

Converting from Tuple to List to modify in Python

I am querying a Oracle database and need some special handling around one column of data that is a clob. I can read in the clobe with .read(). I'd like to write the actual value back to my array. It's a tuple so I must convert to a list, write the value, then convert back to tuple. I am getting the error message: TypeError: 'tuple' object does not support item assignment
My Code:
import cx_Oracle
# USE THIS CONNECTION STRING FOR PRODUCTION
production_username = 'username'
production_password = 'password'
con_string = '%s/%s#hostname:port/orcl' % (production_username, production_password)
con = cx_Oracle.connect(con_string)
cursor = con.cursor()
querystring = ("Select ID, Description from Table")
cursor.execute(querystring)
data = cursor.fetchall()
for currentrow in range(1, len(data)):
description= data[currentrow][1].read()
data = list(data)
data[currentrow][1] = description
data = tuple(data)
con.close()
print data

Try this way
for currentrow in data :
description= currentrow[1].read()
tupled_data= tuple([currentrow[0],description])
print tupled_data

Insert List into oracle db with cx_python returns illegal variable name/number

Using the content of this xml example saved on a local file called test.xml , I'm trying to parse the contents and insert into my database using the following code:
import cx_Oracle
import re
import os
import xml.etree.ElementTree as ET
from ConfigParser import SafeConfigParser
def db_callmany(cfgFile, sql,params):
parser = SafeConfigParser()
parser.read(cfgFile)
dsn = parser.get('odbc', 'dsn')
uid = parser.get('odbc', 'user')
pwd = parser.get('odbc', 'pass')
try:
con = None
con = cx_Oracle.connect(uid , pwd, dsn)
cur = con.cursor()
cur.executemany(sql,params)
con.commit()
except cx_Oracle.DatabaseError, e:
print 'Error %s' % e
sys.exit(1)
finally:
if con:
con.close()
if __name__ == '__main__':
try:
cfgFile='c:\\tests\\dbInfo.cfg'
tree = ET.parse('test.xml')
root = tree.getroot()
mdfList = []
for book in root.findall('book'):
author = book.find('genre').text
title = book.find('price').text
the = str((author,title))
mdfList.append(the)
sql = "INSERT INTO book_table ( GENRE, PRICE )"
db_callmany(cfgFile,sql,mdfList)
except KeyboardInterrupt:
sys.stdout('\nInterrupted.\n')
But executing this code I receive the following error:
Error ORA-01036: illegal variable name/number
Exit code: 1
Not sure what I can be missing here in order to get this code working.

CX_Oracle requires placeholders and then a sequence of dictionaries for executemany rather than a sequence of sequences - so something like:
mdfList = list()
for book in root.findall('book'):
Values = dict()
Values['GENRE'] = book.find('genre').text
Values['PRICE'] = book.find('price').text
mdfList.append(Values)
sql = "INSERT INTO book_table (GENRE, PRICE) VALUES (:GENRE, :PRICE)"
db_callmany(cfgFile, sql, mdfList)