A friend and I created the following script utilizing BeautifulSoup to get the HTML of a job page, then append the job to an array, then a file, then email the job in a human-readable format to ourselves. The script works on Ubuntu, but on my Raspberry Pi, which uses Raspbian, it doesn't work.
The only message I see when running from the terminal is: 'end of file' and 'Start write...' which are lines in the code. There are no error messages when running from the Pi, but nothing gets appended to the array and no emails are sent.
Can someone take a look? Thanks.
import urllib2, email, smtplib, os.path
import cPickle as pickle
from bs4 import BeautifulSoup
class Job:
"""docstring for Job"""
def __init__(self, title, date, url):
self.title = title
self.date = date
self.url = "http://www.forensicfocus.com/"+url
def describJob(self):
return (self.title +" "+ self.date +" "+ self.url)
def createJobsArray():
soup = BeautifulSoup(urllib2.urlopen('http://www.forensicfocus.com/jobs').read())
bigFatString = soup.find_all('a')
#print(bigFatString) #this gets webpage as html. No issues here
findAll = soup.find_all("tr", class_="topic")
jobsArray = []
for section in findAll:
title = section.find("a", class_="topictitle").get_text()
titleEncoded = title.encode('ascii','ignore')
row = section.find_all("td")
date = row[3].find("div").get_text()
url = section.find_all("a")[3].get("href")
job = Job(titleEncoded, date, url)
print "printing job"
print job
print "printing job"
jobsArray.append(job)
return jobsArray
def sendEmail(job):
senderEmail = "sender#example.com"
recipients = ["destination#example.com"]
s = smtplib.SMTP("smtp.gmail.com",587)
s.ehlo()
s.starttls()
s.ehlo()
s.login(senderEmail, 'pass_goes_here')
for job in jobsFilteredByLocation:
msg = email.message_from_string(job.describJob())
msg['Subject'] = "New Job Found: " + job.title
s.sendmail(senderEmail, recipients, msg.as_string())
print "Sending email..."
s.quit()
def saveJobsToDisk(jobs):
with open('hadooken', 'wb') as output:
print "Start write..."
for job in jobs:
print job.title
pickle.dump(job, output)
output.close()
def getJobsFromDisk():
oldJobsArray = []
with open('hadooken', 'rb') as input:
while True:
try:
job = pickle.load(input)
print job.title, "was successfully read from file"
oldJobsArray.append(job)
except EOFError:
print "end of file"
break
return oldJobsArray
input.close()
# SCRIPT STARTS HERE
with open('hadooken', 'ab') as input:
input.close()
locationsArray = ["London"]
jobsArray = createJobsArray()
oldJobsArray = getJobsFromDisk()
jobsFilteredByLocation = []
for job in jobsArray:
for location in locationsArray:
found = job.title.find(location)
if found > 0:
if len(oldJobsArray) > 0:
if any(oldJob.title == job.title for oldJob in oldJobsArray):
print "Job previously found and sent..."
else:
print "adding ", job.title, "to array because it isnt in the old array"
jobsFilteredByLocation.append(job)
else:
print "adding ", job.title, "to array"
jobsFilteredByLocation.append(job)
sendEmail(jobsFilteredByLocation)
mergedArray = oldJobsArray + jobsFilteredByLocation
for job in mergedArray:
print "Job title: ", job.title
saveJobsToDisk(mergedArray)
Related
Hello I need help with my code. It keeps giving me authentication-errors.
Can you check it out for me?
All I needed was the code to authenticate successfully and save the working login in a txt-file and the bad login (wrong password) in another txt-file. It works with smtp but keeps giving me an error on imap.
See the code below.
Thanks
The logins in accounts.txt are in the following format email:password
...
import imaplib
import ssl
import socket
import getpass
import re
import socks
import codecs
import unicodedata
import random
from multiprocessing.pool import ThreadPool
# PROXY_TYPE_HTTP
# PROXY_TYPE_SOCKS5
proxy_type = socks.PROXY_TYPE_HTTP
use_proxies = False
thead_count = 1
use_encrpytion = False
accounts = []
accounts_checked = 0
accounts_valid = []
accounts_invalid = []
proxies = []
def check_account(email, password):
try:
if (use_proxies):
proxy = random.choice(proxies)
proxy_host = proxy.split(':')[0]
proxy_port = int(proxy.split(':')[1])
socks.setdefaultproxy(proxy_type, proxy_host, proxy_port)
socks.wrapmodule(imaplib)
mailserver = imaplib.IMAP4_SSL(('mail.' + re.search('#((\w|\w[\w\-]*?\w)\.\w+)', email).group(1)), 993)
mailserver.login(str(email), str(password))
mailserver.close()
return True
except imaplib.IMAP4.error:
print ("Log in failed.")
return False
def get_status(account):
global accounts_checked, accounts
if (':' not in account):
return False
email = account.split(':')[0]
password = account.split(':')[1]
valid = check_account(email, password)
if (valid):
print("Valid: ", account)
f1 = open("connect.txt", "a+")
f1.write(account)
f1.close()
accounts_valid.append(account)
else:
f2 = open("not_connect.txt", "a+")
f2.write(account)
f2.close()
accounts_invalid.append(account)
accounts_checked += 1
print("(" + str(accounts_checked) + "/" + str(len(accounts)) + ")")
return valid
if __name__ == "__main__":
if (use_proxies):
print("Reading \"proxies.txt\"...")
with open("proxies.txt") as f:
for line in f:
if (':' in line):
proxies.append(line)
print("Found " + str(len(proxies)) + " proxies.")
print("Reading \"accounts.txt\"...")
with codecs.open("accounts.txt", encoding='utf-8') as f:
for line in f:
line = unicodedata.normalize('NFKD', line).encode('ascii','ignore').decode('ascii')
if (':' in line):
accounts.append(line.replace("\n", "").replace("\t", ""))
print("Found " + str(len(accounts)) + " accounts.")
print("Creating thread pool...")
pool = ThreadPool(thead_count)
results = pool.map(get_status, accounts)
pool.close()
pool.join()
print("Done checking, writing output...")
print("Completed!")
...
you should create a minimal example, in my case I cannot log in using
imaplib but I do not wrap with the socket stuff.. Why is the ssl
sockets not automatic?
def get_mail_client(email_address):
print(password)
mail = imaplib.IMAP4_SSL(SMTP_SERVER, SMTP_PORT)
mail.login(email_address, password)
return mail
def start(name):
# Use a breakpoint in the code line below to debug your script.
mailClient = get_mail_client(EMAIL)
status, messages = mailClient.select('INBOX')
print(f'Hi, {name}') # Press Ctrl+F8 to toggle the breakpoint.
print(messages)
print(messages[0])
I am able to use haveibeenpwned to search for 1 account compromise. However, I could not find an option to use the API key to search for compromise of all the email accounts on a domain. (For example. if the domain is xyz.com, I want to search for the compromise of abc#xyz.com, peter.charlie#xyz.com and so on). I am aware of the notification email that I can sign up for. But, that is a lengthy process and I prefer using the API.
So, I wrote a script to search against haveibeenpwned for all the email address of my domain, but it takes very long. I searched through a couple of Github projects, but I did not find any such implementation. Has anyone tried this before?
I have added the code below. I am using Multi threading approach, but still it takes very long, is there any other Optimization strategy I can use? Please help. Thank you.
import requests, json
import threading
from time import sleep
import datetime
import splunklib.client as client
import splunklib.results as results
date = datetime.datetime.now()
from itertools import islice
import linecache
import sys
def PrintException():
exc_type, exc_obj, tb = sys.exc_info()
f = tb.tb_frame
lineno = tb.tb_lineno
filename = f.f_code.co_filename
linecache.checkcache(filename)
line = linecache.getline(filename, lineno, f.f_globals)
print 'EXCEPTION IN ({}, LINE {} "{}"): {}'.format(filename, lineno, line.strip(), exc_obj)
class myThread (threading.Thread):
def __init__(self, threadID, name, list_emails):
threading.Thread.__init__(self)
self.threadID = threadID
self.name = name
self.list_emails = list_emails
def run(self):
i=0
print "Starting " + self.name
for email in self.list_emails:
print i
i=i+1
result = check_pasteaccount(email)
print email
print result
print result
print "Exiting " + self.name
def check_pasteaccount(account):
account = str(account)
result = ""
URL = "https://haveibeenpwned.com/api/v3/pasteaccount/%s?truncateResponse=false" % (account)
# print(URL)
headers= {'hibp-api-key':api_key}
result = ""
try:
r = requests.get(url=URL,headers=headers)
# sleep(2)
status_code = r.status_code
if status_code == 200:
data = r.text
result = []
for entry in json.loads(data.decode('utf8')):
if int((date - datetime.datetime.strptime(entry['Date'], '%Y-%m-%dT%H:%M:%SZ')).days) > 120:
pass
else:
result.append(['Title: {0}'.format(entry['Title']), \
'Source: {0}'.format(['Source']), \
'Paste ID: {0}'.format(entry['Id'])])
if len(result) == 0:
result = "No paste reported for given account and time frame."
else:
paste_result = ""
for entry in result:
for item in entry:
paste_result += str(item) + "\r\n"
paste_result += "\r\n"
result = paste_result
elif status_code == 404:
result = "No paste for the account"
else:
if status_code == 429:
sleep(5)
# print "Limit exceeded, sleeping"
result = check_pasteaccount(account)
else:
result = "Exception"
print status_code
except Exception as e:
result = "Exception"
PrintException()
pass
return result
def split_every(n, iterable):
iterable = iter(iterable)
for chunk in iter(lambda: list(islice(iterable, n)), []):
yield chunk
def main():
print datetime.datetime.now()
# Fetching the list of email addresses from Splunk
list_emails = connect_splunk()
print datetime.datetime.now()
i=0
list_split = split_every(1000,list_emails)
threads=[]
for list in list_split:
i=i+1
thread_name = "Thread" + str(i)
thread = myThread(1, thread_name, list)
thread.start()
threads.append(thread)
# Wait for all the threads to complete
for t in threads:
t.join()
print "Completed Search"
Here's a shorter and maybe more efficient version of your script using the standard multiprocessing library instead of a hand-rolled thread system.
You'll need Python 3.6+ since we're using f-strings.
You'll need to install the tqdm module for fancy progress bars.
You can adjust the number of concurrent requests with the pool size parameter.
Output is written in machine-readable JSON Lines format into a timestamped file.
A single requests session is shared (per-worker), which means less time spent connecting to HIBP.
import datetime
import json
import multiprocessing
import random
import time
import requests
import tqdm
HIBP_PARAMS = {
"truncateResponse": "false",
}
HIBP_HEADERS = {
"hibp-api-key": "xxx",
}
sess = requests.Session()
def check_pasteaccount(account):
while True:
resp = sess.get(
url=f"https://haveibeenpwned.com/api/v3/pasteaccount/{account}",
params=HIBP_PARAMS,
headers=HIBP_HEADERS,
)
if resp.status_code == 429:
print("Quota exceeded, waiting for a while")
time.sleep(random.uniform(3, 7))
continue
if resp.status_code >= 400:
return {
"account": account,
"status": resp.status_code,
"result": resp.text,
}
return {
"account": account,
"status": resp.status_code,
"result": resp.json(),
}
def connect_splunk():
# TODO: return emails
return []
def main():
list_emails = [str(account) for account in connect_splunk()]
datestamp = datetime.datetime.now().isoformat().replace(":", "-")
output_filename = f"accounts-log-{datestamp}.jsonl"
print(f"Accounts to look up: {len(list_emails)}")
print(f"Output filename: {output_filename}")
with multiprocessing.Pool(processes=16) as p:
with open(output_filename, "a") as f:
results_iterable = p.imap_unordered(
check_pasteaccount, list_emails, chunksize=20
)
for result in tqdm.tqdm(
results_iterable,
total=len(list_emails),
unit="acc",
unit_scale=True,
):
print(json.dumps(result, sort_keys=True), file=f)
if __name__ == "__main__":
main()
I wrote a script to send an email if the values match a certain criteria. I'm wanting to send 1 email instead of multiple emails upon every check. I thought I can mitigate by throwing in another function but I can't figure out how to do it. Any ideas on how to accomplish this?
import csv, requests, xmltodict, smtplib, email.utils
from email.mime.text import MIMEText
def sendEmail(host, value, devicename):
# Create the message
msg = MIMEText('This is the body of the message.')
msg['To'] = email.utils.formataddr(('Recipient', 'XXXXXX'))
msg['From'] = email.utils.formataddr(('Author', 'XXXXX'))
msg['Subject'] = 'Simple test message'
server = smtplib.SMTP('XXXXXXX')
server.set_debuglevel(True) # show communication with the server
try:
server.sendmail('XXXXXX', ['XXXXXX'], msg.as_string())
finally:
server.quit()
def check(hostIP, value):
xml = """<?xml version="1.0" encoding="iso-8859-1"?>"""
headers = {'Content-Type': 'application/xml'}
response = requests.post('http://' + hostIP + '/RPC2', data=xml, headers=headers).text
doc = xmltodict.parse(response)
uptime = str(doc['response'])
maxtime = '300'
time = str(uptimeValue)
day = time // (24 * 3600)
if day >= maxtime:
print 'it is'
sendEmail(str(hostIP), str(value), str(devicename))
else:
print "it is not!"
def main():
try:
with open('list.csv', 'r') as file:
reader = csv.DictReader(file)
for row in reader:
check(row['Host'], row['Value'])
except Exception as error:
print ValueError("Could not properly read the csv file")
sys.exit(0)
if __name__ == "__main__":
main()
Replace
def sendEmail(message):
and
if day >= maxtime:
print 'it is'
return (str(hostIP), str(value), str(devicename))
and
device_list = []
for row in reader:
result = check(row['Host'], row['Value'])
if result:
device_list.append(', '.join(result))
sendEmail('\n'.join(device_list))
Instead of using check to send the email, use it to add an item to go into the message body, probably in an array or dictionary. Then, after you are done processing your file, you can use that collected information to build the body of your email to give to a single call to sendEmail.
That means you don't call sendEmail from inside of check; you call it from main once you are done with the file.
I've written a Python script to automatically send some information to my friends. I used SMTPlib, it works well if I only sent to me or one additional email.
When I try to send to 17 emails, (including my sender email), then it shows in sent mail on web-based Gmail. I saw that the mail was sent but I didn't receive it. Only the first recipient received the email.
If I reply to all from that mail, then everyone got only that reply.
I can't figure out why they didn't receive it when I sent it from script, I ask my friend check spam, but she didn't find anything.
This is my code:
#!/usr/bin/env python
import smtplib
import csv
from datetime import datetime, timedelta
SMTP_SERVER = 'smtp.gmail.com'
SMTP_PORT = 587
sender = 'MYBOT#gmail.com'
password = None
with open('pass', 'rt') as f:
password = f.read().strip('\n')
def send_mail(recipient, subject, body):
"""
Send happy bithday mail
"""
headers = ["From: " + sender,
"Subject: " + subject,
"To: " + recipient,
"MIME-Version: 1.0",
"Content-Type: text/html"]
headers = "\r\n".join(headers)
smtp = smtplib.SMTP(SMTP_SERVER, SMTP_PORT)
smtp.ehlo()
smtp.starttls()
smtp.ehlo
smtp.login(sender, password)
body = "" + body +""
smtp.sendmail(sender, recipient, headers + "\r\n\r\n" + body)
print "Sent to ",
print recipient
smtp.quit()
def send_happybirthday(recipient):
body = """Happy birthday to you!
\n<br/>From C2k8pro with love
"""
subject ='[BirthReminder] Happy birthday to you! from C2k8pro'
send_mail(recipient, subject, body)
def send_notification(all_mails, names):
body = """Tomorrow is birthday of %s""" % names
send_mail(all_mails, body, body)
def test_send_mail():
notify_body = """Tomorrow is birthday of """
recipients = ['MYBOT#gmail.com']
today = datetime.now()
format = "%d-%m-%Y"
print today
today_in_str = datetime.strftime(today, format)
def read_csv():
FILENAME = 'mails.csv'
reader = csv.reader(open(FILENAME, 'rt'), delimiter=',')
today = datetime.now()
one_day = timedelta(days=1)
tomorrow = today + one_day
all_mails = []
str_format = "%d/%m"
str_today = today.strftime(str_format)
str_tomorrow = tomorrow.strftime(str_format)
print 'Today is ', str_today
tomorrow_birth = []
for row in reader:
name = row[1].strip()
dob = row[2]
dmy = dob.split("/")
mail = row[3]
all_mails.append(mail)
#TODO fix dob with only 1 digit
birth_date = dmy[0] + "/" + dmy[1]
if str_today == birth_date:
print 'Happy birthday %s' % name
try:
send_happybirthday(mail)
except Exception, e:
print e
elif str_tomorrow == birth_date:
tomorrow_birth.append(name)
print "Tomorrow is %s's birthday" % name
# Remove empty string
all_mails = filter(None, all_mails)
print 'All mails: ', len(all_mails)
str_all_mails = ', '.join(all_mails)
if tomorrow_birth:
all_tomorrow = ', '.join(tomorrow_birth)
send_notification(str_all_mails, all_tomorrow)
def main():
read_csv()
if __name__ == "__main__":
main()
Can anyone explain this. Thanks!
I found solution from here
Send Email to multiple recipients from .txt file with Python smtplib
I passed a string contain all recipients separated by comma to msg['To'] and sendmail().
It's true for msg['To'] but with sendmail, I have to use a list.
I just get the error:
AttributeError: SecondLife instance has no attribute 'sendAMail'
Whats wrong?
(I checked the formating and this is not the error.
I checked the syntax and also not the error.)
What in the script happens is that an url gets open with cookies and i want some information from it.
import urllib2, cookielib, re
import ClientForm
import re
import smtplib
kurse = ['Entwicklung von Multimediasystemen', 'Computergrafik', 'Gestaltung von Multimediasystemen', 'Verteilte Systeme']
class SecondLife:
def __init__(self, usernames, password):
self.username = usernames
self.password = password
self.url = 'https://lsf.htw-berlin.de/qisserver/rds?state=user&type=0&application=QISPOS'
cookiejar = cookielib.LWPCookieJar()
cookiejar = urllib2.HTTPCookieProcessor(cookiejar)
# debugger = urllib2.HTTPHandler(debuglevel=1)
opener = urllib2.build_opener(cookiejar)
urllib2.install_opener(opener)
def sendAMail(self, smtp_server, user, password, listener, subject, text):
smtp = smtplib.SMTP(smtp_server)
smtp.starttls()
smtp.login(user,password)
msg = "SUBJECT: " + subject + "\n\n" + text
smtp.sendmail("bln.schade#gmail.com", listener, msg)
smtp.quit()
def login(self):
response = urllib2.urlopen(self.url)
forms = ClientForm.ParseResponse(response, backwards_compat=False)
# forms[0] is 'GET', forms[1] is 'POST'
form = forms[0]
try:
form['username'] = self.username
form['password'] = self.password
except Exception, e:
print 'The following error occured: \n"%s"' % e
print
print 'A good idea is to open a browser and see if you can log in from there.'
print 'URL:', self.url
exit()
self.page = urllib2.urlopen(form.click('submit')).read()
def friends_online(self):
self.login()
final = ""
final_asi = ""
leistungsstand = ""
match = re.search(r"asi=\w*\d*\"", self.page)
if match:
final = match.group()
final_asi = re.sub("asi=", "", final)
final_asi = re.sub("\"", "", final_asi)
print "vorher: " + final
print "nachher: " + final_asi
leistungsstand_url = "https://lsf.htw-berlin.de/qisserver/rds?state=htmlbesch&application=sospos&moduleParameter=Student&navigationPosition=functions%2Cnotenspiegel&breadcrumb=notenspiegel&topitem=functions&subitem=notenspiegel&asi=" + final_asi
leistungsstand = urllib2.urlopen(leistungsstand_url).read()
else:
print "not match"
# Ausloggen
logout = "https://lsf.htw-berlin.de/qisserver/rds?state=user&type=4&re=last&menuid=logout&category=auth.logout"
urllib2.urlopen(logout).read()
website = open("lsf.html", "w")
website.write(leistungsstand)
website.close()
for kurs in kurse:
print kurs
if (re.search(kurs, "fajfjsjj Entwicklung von Multimediasystemen hahahah")):
self.sendAMail("smtp.googlemail.com", "user", "passw", "bln.schade#gmail.com", "kurs" , "Eine neue Note ist im LSF eingetragen.")
#self.final_asi.replace(new, "asi=","")
#asi[0].replace("\"","")
#print "Final " + asi
SL = SecondLife('xyz', 'xyz')
SL.friends_online()
Works for me: printing out self.sendAMail from within an instance gives
<bound method SecondLife.sendAMail of <__main__.SecondLife instance at 0x101d91e18>>
I think it is a formatting issue, though. If I copy and paste your code and look at the whitespace, I see mixed use of spaces and tabs. In particular:
In [20]: [line for line in d if 'def' in line]
Out[20]:
[' def __init__(self, usernames, password):\n',
' \tdef sendAMail(self, smtp_server, user, password, listener, subject, text):\n',
' def login(self):\n',
' def friends_online(self):\n']
The \t before def sendAMail looks very suspicious. I'm 75% sure the inconsistent whitespace is what's causing the problem. Try running your script using python -tt scriptname.py, which will throw an error about inconsistent tab usage.