I'm setting up a program that extracts the date, sender, subject and body of emails in my outlook. However, when I run the code it gives me the below error:
Traceback (most recent call last):
File "C:\Users\zaballgl\Documents\Adhoc\2019\April\ETL_MetricsEmailOutlook.py", line 83, in <module>
cursor.execute("INSERT INTO dbo.BSO_metricsEmailReports([Start_Date],[Name],[Subject],[Body])values(?,?,?,?)",row['Start_Date'],row['Name'],row['Subject'],row['Body'])
pyodbc.ProgrammingError: ('Invalid parameter type. param-index=1 param-type=CDispatch', 'HY105')
This is my code for extracting data from my outlook:
import win32com.client
import pandas as pd
import datetime
import numpy as np
import pyodbc
outlook = win32com.client.Dispatch("Outlook.Application").GetNamespace("MAPI")
inbox = outlook.Folders('email#outlook.com').Folders('Inbox')
messages = inbox.Items
message = messages.GetFirst()
rec_time = message.CreationTime
body_content = message.body
subj_line = message.subject
sender = message.Sender
year=[]
month=[]
day=[]
hour=[]
minute=[]
subject=[]
sender=[]
body = []
while message:
###This iterates every format of the message.CreationTime and append them to the list above
year.append(message.CreationTime.year)
month.append(message.CreationTime.month)
day.append(message.CreationTime.day)
hour.append(message.CreationTime.hour)
minute.append(message.CreationTime.minute)
## Iterates every subject and append them to the subject variable list
subject.append(message.subject)
## Iterates every sender name and append them to the sender variable list
sender.append(message.Sender)
## Iterates every sender name and append them to the sender variable list
body.append(message.body)
## Goes to the next email
message = messages.GetNext()
## This saves all the information to a context manager
#------COLUMNS FOR THE TABLE---------------#
#StartDate
date = pd.DataFrame({'year':year,'month':month,'day':day,'hour':hour,'minute':minute}) # Had to do this to bypass this error: ValueError: Tz-aware datetime.datetime cannot be converted to datetime64 unless utc=True
startDate = pd.to_datetime(date) # Using the above variable this converts this to dtype: datetime64[ns]
#Subject
subject = pd.Series(subject) # just a series of subject data
#Sender
sender = pd.Series(sender) # just a series of sender data
#Body
body = pd.Series(body) # just a series of sender data
df2 = pd.DataFrame({'Start_Date':startDate,'Name':sender, 'Subject': subject, 'Body':body})
And this is my code to transfer them to my MS SQL:
connStr = pyodbc.connect('DRIVER={ODBC Driver 13 for SQL Server}; Server=someservername;DATABASE=somedatabase;UID=someID;PWD=somepassword#')
cursor = connStr.cursor()
deleteTable = "DELETE FROM dbo.BSO_metricsEmailReports"
cursor.execute(deleteTable)
for index,row in df2.iterrows():
cursor.execute("INSERT INTO dbo.BSO_metricsEmailReports([Start_Date], [Name],[Subject], [Body])values(?,?,?,?)",row['Start_Date'],row['Name'],row['Subject'],row['Body'] )
connStr.commit()
cursor.close()
connStr.close()
They would be sent to a table in my MS SQL 2014 with the below design:
**Column Name** | **Data Type**
Start_Date | datetime
Name | nchar(300)
Subject | nchar(300)
Body | nchar(300)
I'd check the execute line...
I use the ? for each parameter in the query and set it up:
sql_rollup =
'''
SELECT ID,FIRSTNAME, LASTNAME, .USERNAME, numSessions
FROM SESSION INNER JOIN
PERSONNEL ON SESSION.ID = PERSONNEL.ID
WHERE (SessionStartDT between ? AND ?) AND (SiteID = ?)
'''
Then I execute the above like this:
con = pyodbc.connect(
Trusted_connection='Yes',
Driver='{SQL Server}',
Server=myConfig["database"]["hostname"] + ',' + myConfig["database"]["port"],
Database=myConfig["database"]["database"]
)
con.autocommit = True
cur=con.cursor()
parms = (str(dateFirst), str(dateLast), siteID)
cur.execute(sql_rollup,parms)
Note the params is set up as a list and also (and I suspect this is where the problem lies), I convert the datatime values for dateFirst and dateLast to strings.
pyodbc doesn't understand Python objects. The database on the otherhand can automatically interpret strings and convert them into date/time values.
Does this help?
Related
This question already has answers here:
How can I parse a time string containing milliseconds in it with python?
(7 answers)
Closed 14 days ago.
i am reading a entrance time from sqlite database to calaulate the duration of the car stay.
the datetime is currenly able to insert and retrive but when formated to do calualtion it keeps give an error when using the datetime.strptime funtion.EnterTime is stored as a text in the sqlite database.
import sqlite3
import smtplib
from datetime import datetime, timedelta
# Connect to the database
conn = sqlite3.connect("py.db")
# Get current time
current_time = datetime.now()
# Define the carplate number
carplate = "SJJ4649G"
# Check if the carplate already exists in the database
cursor = conn.cursor()
query = "SELECT * FROM entrance WHERE carplate = ?"
cursor.execute(query, (carplate,))
result = cursor.fetchall()
# If the carplate already exists, send an email
if len(result) > 0:
# Get the email address from the gov_info table
query = "SELECT email FROM gov_info WHERE carplate = ?"
cursor.execute(query, (carplate,))
email_result = cursor.fetchall()
# Get the entrance time from the entrance table
query = "SELECT EnterTime FROM entrance WHERE carplate = ?"
cursor.execute(query, (carplate,))
entrance_time_str = cursor.fetchone()[0]
print (entrance_time_str)
entrance_time = datetime.strptime(entrance_time_str, "%Y-%m-%d %H:%M:%S")
# Calculate the cost
delta = current_time - entrance_time
cost = delta.total_seconds() / 3600 * 10 # 10 is the hourly rate
# Email details
email = "testcsad69#gmail.com"
password = "ufwdiqcfepqlepsn"
send_to = email_result[0][0]
subject = "Parking Fees"
message = f"The cost for parking the car with plate number {carplate} is ${cost:.2f}. The entrance time was {entrance_time} and the current time is {current_time}."
# Send the email
smtp = smtplib.SMTP('smtp.gmail.com', 587)
smtp.ehlo()
smtp.starttls()
smtp.login(email, password)
smtp.sendmail(email, send_to, f"Subject: {subject}\n\n{message}")
smtp.quit()
# If the carplate does not exist, insert it into the database
else:
query = "INSERT INTO entrance (carplate, EnterTime) VALUES (?, ?)"
cursor.execute(query, (carplate, current_time))
conn.commit()
# Close the connection
cursor.close()
conn.close()
i have printed the entrance time out and it matches the data shown in the database.
i have also tried to remove the fromating that would not let me do the calulations.
this is the error i get
2023-02-06 16:07:46.640395
Traceback (most recent call last):
File "/media/pi/6134-E775/payment.py", line 32, in <module>
entrance_time = datetime.strptime(entrance_time_str, "%Y-%m-%d %H:%M:%S")
File "/usr/lib/python3.7/_strptime.py", line 577, in _strptime_datetime
tt, fraction, gmtoff_fraction = _strptime(data_string, format)
File "/usr/lib/python3.7/_strptime.py", line 362, in _strptime
data_string[found.end():])
ValueError: unconverted data remains: .640395
You need to append .%f to take microseconds:
entrance_time_str = '2023-02-06 16:07:46.640395' # HERE --v
entrance_time = datetime.strptime(entrance_time_str, "%Y-%m-%d %H:%M:%S.%f")
>>> entrance_time_str
'2023-02-06 16:07:46.640395'
>>> entrance_time
datetime.datetime(2023, 2, 6, 16, 7, 46, 640395)
I'm downloading a data from an API and storing it in SQLite db. I want to implement the process using "multithreading". Can someone please help me with how to implement it.
I found a library but getting an error. below is the code.
import sqlite3
import os
import pandas as pd
from sodapy import Socrata
import concurrent.futures
dbPath = 'folder where db exists'
dbName = 'db file name'
## Setup connection & cursor with the DB
dbConn = sqlite3.connect(os.path.join(dbPath, dbName), check_same_thread=False)
## Setup the API and bring in the data
client = Socrata("health.data.ny.gov", None)
## Define all the countys to be used in threading
countys = [all 62 countys in New York]
varDict = dict.fromkeys(countys, {})
strDataList = ['test_date', 'LoadDate']
intDataList = ['new_positives', 'cumulative_number_of_positives', 'total_number_of_tests', 'cumulative_number_of_tests']
def getData(county):
## Check if table exists
print("Processing ", county)
varDict[county]['dbCurs'] = dbConn.cursor()
varDict[county]['select'] = varDict[county]['dbCurs'].execute('SELECT name FROM sqlite_master WHERE type="table" AND name=?', (county,) )
if not len(varDict[county]['select'].fetchall()):
createTable(county)
whereClause = 'county="'+county+'"'
varDict[county]['results'] = client.get("xdss-u53e", where=whereClause)
varDict[county]['data'] = pd.DataFrame.from_records(varDict[county]['results'])
varDict[county]['data'].drop(['county'], axis=1, inplace=True)
varDict[county]['data']['LoadDate'] = pd.to_datetime('now')
varDict[county]['data'][strDataList] = varDict[county]['data'][strDataList].astype(str)
varDict[county]['data']['test_date'] = varDict[county]['data']['test_date'].apply(lambda x: x[:10])
varDict[county]['data'][intDataList] = varDict[county]['data'][intDataList].astype(int)
varDict[county]['data'] = varDict[county]['data'].values.tolist()
## Insert values into SQLite
varDict[county]['sqlQuery'] = 'INSERT INTO ['+county+'] VALUES (?,?,?,?,?,?)'
varDict[county]['dbCurs'].executemany(varDict[county]['sqlQuery'], varDict[county]['data'])
dbConn.commit()
# for i in dbCurs.execute('SELECT * FROM albany'):
# print(i)
def createTable(county):
sqlQuery = 'CREATE TABLE ['+county+'] ( [Test Date] TEXT, [New Positives] INTEGER NOT NULL, [Cumulative Number of Positives] INTEGER NOT NULL, [Total Number of Tests Performed] INTEGER NOT NULL, [Cumulative Number of Tests Performed] INTEGER NOT NULL, [Load date] TEXT NOT NULL, PRIMARY KEY([Test Date]))'
varDict[county]['dbCurs'].execute(sqlQuery)
# for _ in countys:
# getData(_)
# x = countys[:5]
with concurrent.futures.ThreadPoolExecutor() as executor:
# results = [executor.submit(getData, y) for y in x]
executor.map(getData, countys)
getData is the function which brings in the data county wise and loads into the db. Countys is a list of all the countys. I am able to do it synchronously but would like to implement multithreading.
The for loop to do it synchronously (which works) is
for _ in countys:
getData(_)
The error message is
ProgrammingError: SQLite objects created in a thread can only be used in that same thread. The object was created in thread id 8016 and this is thread id 19844.
You might find this useful
sqlite.connect(":memory:", check_same_thread=False)
I have a problem where I am trying to call a specific field from the data recovered by the self.results variable from the sqlite3 login database, although I am unable to do this as I believe that the fetched data is not in an array format and therefore the system is unable to use that field, I got rid of all the " ' ", "(", ")" but I do not know what to do now to convert this text file into an array so that a field can be fetched and printed.
Could you help me?
while True:
username = self.usernameEntry.get()
password = self.passwordEntry.get()
conn = sqlite3.connect("database.db")
cursor = conn.cursor()
findUser = ("SELECT * FROM students WHERE CardNumberID = ? AND Password = ?")
cursor.execute(findUser, [(username), (password)])
self.results = cursor.fetchone()
fetchedResults = str(self.results)
fetchedResults = fetchedResults.replace('(', '')
fetchedResults = fetchedResults.replace(')', '')
fetchedResults = fetchedResults.replace("'", '')
fetchedResults.split(',')
print(fetchedResults[2])
print(self.results)
Here are the results that I get:
The results are in an "array" format, but you then explicitly convert the whole thing to a string. Don't do that.
My output is coming : total records in staging table are (3L,)
I only want number n brackets and how can i remove L ,
Im converting query to str because i am not able to concanete part1 and msg
#!/usr/bin/env python
import csv
import MySQLdb
import pysftp
import smtplib
import sys
#connection to Database
conn = MySQLdb.connect(host="XXXXX", # The Host
user="XXXXX", # username
passwd="XXXXX", # password
db="XXXXX") # name of the data base
part1 = """From: XXXXX
To: To Person <XXXXX.com>
MIME-Version: 1.0
Content-type: text/html
Subject: SMTP HTML e-mail test
Toatal records in Staging Table inserted:
"""
sender = 'XXXXX'
receivers = ['XXXXX','XXXXX']
x = conn.cursor()
query = "select count(*) as toatal_records_inserted from fpwbs_feed_stg"
x.execute(query)
msg = ""
for row in x.fetchall():
msg += str(row)
msg=part1+msg
smtpObj=smtplib.SMTP('XXXXX', 25)
smtpObj.sendmail(sender, receivers, msg)
Do this to get the first item in the tuple:
for row in x.fetchall():
msg += str(row[0])
I am working on my first Python script to check email then parse out some data then insert into a Microsoft sql database. I am having problems with how to insert the saved variable t.
import imaplib
import email
import re
import _mssql
m = imaplib.IMAP4_SSL("imap.gmail.com", 993)
m.login("username","password$")
m.select('"Fire_Dispatch"')
result, data = m.search(None, "ALL") # search all email and return uids
if result == 'OK':
for num in data[0].split():
result, data = m.fetch(num,"(BODY[TEXT])")
if result == 'OK':
email_message = email.message_from_string(data[0] [1].decode('utf-8')) # raw email text including headers
p = str(email_message)
a = re.search("ALLCFD(.*)",p)
t = (a.groups(1))
print (t)
conn = _mssql.connect(server='server', user='username', password='password', database='CFD_Calls')
conn.execute_non_query("INSERT INTO calls (call_date_time,call)VALUES('3',t)")
m.close()
m.logout()
You'll need to provide the t variable into the sql execution. One way to do this:
conn.execute_non_query("INSERT INTO calls (call_date_time,call) VALUES('3',?)", (t,))
I'm assuming t will be a string. Note that when provided to the sql command, the variable is enclosed in a tuple. This is because the command expects the argument to be a sequence.