I have some servers that create automated emails which all pass through a Postfix MTA. The software that generates the email does not strictly follow RFCs, and sometimes generates emails with duplicate message-ID headers. The software cannot be changed, so I am trying to intercept and fix these messages on their way through the MTA.
I have a milter daemon written in Python that is attempting to remove duplicate message IDs from inbound messages.
The code is below:
import Milter
import logging
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
file_handler = logging.FileHandler('/var/log/milter.log')
file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
logger.addHandler(file_handler)
seen_message_ids = set()
class MessageIDMilter(Milter.Base):
def __init__(self):
self.id = Milter.uniqueID()
#Milter.noreply
def connect(self, IPname, family, hostaddr):
logger.debug("Milter connected to %s family %s at address %s" % (IPname, family, hostaddr))
self.IPname = IPname
self.family = family
self.hostaddr = hostaddr
return Milter.CONTINUE
#Milter.noreply
def header(self, name, hval):
logger.debug("Received header %s with value %s" % (name, hval))
if name.lower() == "message-id":
logger.debug("Found message ID: %s" % hval)
if hval in seen_message_ids:
logger.debug("Deleting duplicate message ID: %s" % hval)
try:
self.chgheader(name, 1, "")
except Exception as e:
logger.error("Error removing from: %s error message: %s" % (name, e))
else:
seen_message_ids.add(hval)
return Milter.CONTINUE
#Milter.noreply
def eoh(self):
logger.debug("Reached end of headers")
return Milter.ACCEPT
if __name__ == "__main__":
logger.debug("Script started OK")
Milter.factory = MessageIDMilter
Milter.runmilter("message-id-milter", "inet:10001#localhost", 0)
The script runs and can be called from Postfix. When attempting to delete the duplicate header with chgheader, the following error is thrown:
2023-02-03 18:22:44,983 ERROR Error removing from: Message-ID error message: cannot change header
I cannot see anything wrong with this request, nor any other method to remove the header. The docs suggest this should work: https://pythonhosted.org/pymilter/milter_api/smfi_chgheader.html
def get_email_body(self, email):
user = self.email_usr
password = self.app_email_pwd
connection = imaplib.IMAP4_SSL('imap.gmail.com')
connection.login(user, password)
connection.list()
connection.select('"INBOX"')
time.sleep(5)
result_search, data_search = connection.search(None, 'TO', email, 'SUBJECT', '"some subject"')
required_email = data_search[0]
result_fetch, data_fetch = connection.fetch(required_email, '(RFC822)')
email_body_string = data_fetch[0][1].decode('utf-8')
confirmation_link = self.parse_confirmation_link(email_body_string)
return confirmation_link
This function works like 2 times of 4 runs. Usually it fails with:
self = <imaplib.IMAP4_SSL object at 0x7fa853614b00>, name = 'FETCH'
tag = b'JAAL5'
def _command_complete(self, name, tag):
# BYE is expected after LOGOUT
if name != 'LOGOUT':
self._check_bye()
try:
typ, data = self._get_tagged_response(tag)
except self.abort as val:
raise self.abort('command: %s => %s' % (name, val))
except self.error as val:
raise self.error('command: %s => %s' % (name, val))
if name != 'LOGOUT':
self._check_bye()
if typ == 'BAD':
raise self.error('%s command error: %s %s' % (name, typ, data))
E imaplib.error: FETCH command error: BAD [b'Could not parse command']
/usr/lib/python3.4/imaplib.py:964: error
My suggestion was that sometimes email isn't delivered at the moment of .search that's why I added time.sleep (I'm searching for the email immediately after it was sent).
Else I did try search while result_fetch is not 'OK' but is also didn't help.
Any other suggestions?
oooops, my suggestion was correct, but time.sleep was in the incorrect place. Moved sleep before connection and all go smooth
I'm trying to trap the following error in a try/exception block, but as this is a custom module that is generating the error - not generating a standard error such as ValueError for example. What is the correct way to catch such errors?
Here is my code:
try:
obj = IPWhois(ip_address)
except Exception(IPDefinedError):
results = {}
else:
results = obj.lookup()
The most obvious way:
except IPDefinedError:
gives:
NameError: name 'IPDefinedError' is not defined
The error returned that I want to check for is:
ipwhois.exceptions.IPDefinedError
ipwhois.exceptions.IPDefinedError: IPv4 address '127.0.0.1' is already defined as 'Private-Use Networks' via 'RFC 1918'.
The issue here is the import!
I had the import as
from ipwhois import IPWhois
but I also needed
import ipwhois
So the following works:
try:
obj = IPWhois(ip_address)
except ipwhois.exceptions.IPDefinedError:
results = {}
else:
results = obj.lookup()
Here is a quick recap. Yes, the error in your question did look like it was likely related to an import issue (as per my comment ;) ).
from pprint import pprint as pp
class IPDefinedError(Exception):
"""Mock IPDefinedError implementation
"""
pass
class IPWhois(object):
"""Mock IPWhois implementation
"""
def __init__(self, ip_address):
if ip_address == "127.0.0.1":
raise IPDefinedError(
"IPv4 address '127.0.0.1' is already defined as 'Private-Use Networks' via 'RFC 1918'.")
self._ip_address = ip_address
def lookup(self):
return "RESULT"
def lookup(ip_address):
""" calculates IPWhois lookup result or None if unsuccessful
:param ip_address:
:return: IPWhois lookup result or None if unsuccessful
"""
result = None
try:
obj = IPWhois(ip_address)
result = obj.lookup()
except IPDefinedError as e:
msg = str(e)
print("Error received: {}".format(msg)) # do something with msg
return result
if __name__ == '__main__':
results = map(lookup, ["192.168.1.1", "127.0.0.1"])
pp(list(results)) # ['RESULT', None]
I have the following block of Python code talking to DynamoDB on AWS:
try:
response = conn.batch_write_item(batch_list)
except Exception ,e:
try:
mess = e.message
except:
mess = "NOMESS"
try:
earg0 = e.args[0]
except:
earg0 = "NOEARG0"
try:
stre = str(e)
except:
stre = "NOSTRE"
print "mess = '%s'" % mess
print "earg0 = '%s'" % earg0
print "stre = '%s'" % stre
What I get is this:
mess = ''
earg0 = 'NOEARG0'
stre = 'DynamoDBValidationError: 400 Bad Request {'message': 'Item size has exceeded the maximum allowed size', '__type': 'com.amazon.coral.validate#ValidationException'}'
What I need to somehow reliably extract the message string such as 'Item size has exceeded the maximum allowed size' from e. How can I do it?
I'm assuming you're using boto to access DynamoDB.
Here is the JSONResponseError (supersuperclass of DynamoDBValidationError) __init__ method:
self.status = status
self.reason = reason
self.body = body
if self.body:
self.error_message = self.body.get('message', None)
self.error_code = self.body.get('__type', None)
if self.error_code:
self.error_code = self.error_code.split('#')[-1]
Wild guess: I would go with e.error_message to get 'Item size has exceeded ...'.
You can also print all attributes (and their values) of e:
for attr in dir(e):
print "e[%r] = '''%s'''" % (attr, getattr(e, attr))
Take e.body, You will get the error as a dictionary.
example:
{u'message': u'The conditional request failed', u'__type': u'com.amazonaws.dynamodb.v20120810#ConditionalCheckFailedException'}
From this easily you will get message.
I have couple of classes in jira.py, providing 2 for sample
class JiraCommand:
name = "<default>"
aliases = []
summary = "<--- no summary --->"
usage = ""
mandatory = ""
commands = None
def __init__(self, commands):
self.commands = commands
def dispatch(self, logger, jira_env, args):
"""Return the exit code of the whole process"""
if len(args) > 0 and args[0] in ("--help", "-h"):
logger.info("")
alias_text = ''
first_alias = True
for a in self.aliases:
if first_alias:
if len(self.aliases) == 1:
alias_text = " (alias: " + a
else:
alias_text = " (aliases: " + a
first_alias = False
else:
alias_text += ", " + a
if not first_alias:
alias_text += ")"
logger.info("%s: %s%s" % (self.name, self.summary, alias_text))
if self.usage == "":
opts = ""
else:
opts = " [options]"
logger.info("")
logger.info("Usage: %s %s %s%s" % \
(sys.argv[0], self.name, self.mandatory, opts))
logger.info(self.usage)
return 0
results = self.run(logger, jira_env, args)
if results:
return self.render(logger, jira_env, args, results)
else:
return 1
def run(self, logger, jira_env, args):
"""Return a non-zero object for success"""
return 0
def render(self, logger, jira_env, args, results):
"""Return 0 for success"""
return 0
and a second class in the same file "jira.py"
class JiraCat(JiraCommand):
name = "cat"
summary = "Show all the fields in an issue"
usage = """
<issue key> Issue identifier, e.g. CA-1234
"""
def run(self, logger, jira_env, args):
global soap, auth
if len(args) != 1:
logger.error(self.usage)
return 0
issueKey = args[0]
try:
jira_env['fieldnames'] = soap.service.getFieldsForEdit(auth, issueKey)
except Exception, e:
# In case we don't have edit permission
jira_env['fieldnames'] = {}
try:
return soap.service.getIssue(auth, issueKey)
except Exception, e:
logger.error(decode(e))
def render(self, logger, jira_env, args, results):
# For available field names, see the variables in
# src/java/com/atlassian/jira/rpc/soap/beans/RemoteIssue.java
fields = jira_env['fieldnames']
for f in ['key','summary','reporter','assignee','description',
'environment','project',
'votes'
]:
logger.info(getName(f, fields) + ': ' + encode(results[f]))
logger.info('Type: ' + getName(results['type'], jira_env['types']))
logger.info('Status: ' + getName(results['status'], jira_env['statuses']))
logger.info('Priority: ' + getName(results['priority'], jira_env['priorities']))
logger.info('Resolution: ' + getName(results['resolution'], jira_env['resolutions']))
for f in ['created', 'updated',
'duedate'
]:
logger.info(getName(f, fields) + ': ' + dateStr(results[f]))
for f in results['components']:
logger.info(getName('components', fields) + ': ' + encode(f['name']))
for f in results['affectsVersions']:
logger.info(getName('versions', fields) + ': ' + encode(f['name']))
for f in results['fixVersions']:
logger.info('Fix Version/s:' + encode(f['name']))
# TODO bug in JIRA api - attachmentNames are not returned
#logger.info(str(results['attachmentNames']))
# TODO restrict some of the fields that are shown here
for f in results['customFieldValues']:
fieldName = str(f['customfieldId'])
for v in f['values']:
logger.info(getName(fieldName, fields) + ': ' + encode(v))
return 0
Now, JiraCat is using JiraCommand as an argument
How can i use JiraCat to get live results
here is what i tried:
>>> from jira import JiraCommand
>>> dir(JiraCommand)
['__doc__', '__init__', '__module__', 'aliases', 'commands', 'dispatch', 'mandatory', 'name', 'render', 'run', 'summary', 'usage']
>>> jcmd = JiraCommand("http://jira.server.com:8080")
>>> from jira import JiraCat
>>> dir(JiraCat)
['__doc__', '__init__', '__module__', 'aliases', 'commands', 'dispatch', 'mandatory', 'name', 'render', 'run', 'summary', 'usage']
>>> jc = JiraCat(jcmd)
>>> print jc
<jira.JiraCat instance at 0x2356d88>
>>> jc.run("-s", "cat", "QA-65")
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "jira.py", line 163, in run
logger.error(self.usage)
AttributeError: 'str' object has no attribute 'error'
DonCallisto has got it right.
JiraCat's run method takes three arguments (logger, jira_env, args); the first one is supposed to be a logger object but you're passing a string ("-s").
So the error that reports a string (logger="-s") has no "error" attribute means just that.
Your comment about the command line (subprocess.Popen(['python', 'jira', '-s', 'jira.server.com:8080';, 'catall', 'JIRA-65'])) is not the same as calling the run() method with the same arguments. Have a look at the bottom of jira.py and see what it does with sys.argv...
Edit (1):
Having read the code, the following python should replicate your command line call. It's a bit complicated, and misses out all the exception handling and logic in jira.py itself, which could get flaky, and I can't test it here.
import jira
import os
com = jira.Commands()
logger = jira.setupLogging()
jira_env = {'home':os.environ['HOME']}
command_name = "cat"
my_args = ["JIRA-65"]
server = "http://jira.server.com:8080" + "/rpc/soap/jirasoapservice-v2?wsdl"
class Options:
pass
options = Options()
#You might want to set options.user and options.password here...
jira.soap = jira.Client(server)
jira.start_login(options, jira_env, command_name, com, logger)
com.run(command_name, logger, jira_env, my_args)