I am using python logging module. I am initialising file having following data
def initialize_logger(output_dir):
'''
This initialise the logger
:param output_dir:
:return:
'''
root = logging.getLogger()
root.setLevel(logging.INFO)
format = '%(asctime)s - %(levelname)-8s - %(message)s'
date_format = '%Y-%m-%d %H:%M:%S'
if 'colorlog' in sys.modules and os.isatty(2):
cformat = '%(log_color)s' + format
f = colorlog.ColoredFormatter(cformat, date_format,
log_colors={'DEBUG': 'green', 'INFO': 'green',
'WARNING': 'bold_yellow', 'ERROR': 'bold_red',
'CRITICAL': 'bold_red'})
else:
f = logging.Formatter(format, date_format)
#ch = logging.FileHandler(output_dir, "w")
ch = logging.StreamHandler()
ch.setFormatter(f)
root.addHandler(ch)
As there is only one streamHandler, But I am getting two prints on my console as
INFO:root:clearmessage:%ss1=00
2017-12-21 17:07:20 - INFO - clearmessage:%ss1=00
INFO:root:clearmessage:%ss2=00
2017-12-21 17:07:20 - INFO - clearmessage:%ss2=00
Every message is printed as Root and Info. Any idea Why I am getting two prints. In the above code you can ignore color code.
You have two handlers. Clear the handlers before you add a new one:
root.handlers = [] # clears the list
root.addHandler(ch) # adds a new handler
I get log messages with the same date when I print them to the console (or logfile). But the time-out between messages is two seconds. Here is my code
folder = "logs"
log_name = {}.log
file_name = os.path.join(folder, log_name)
date_format = "%Y-%m-%d_%H:%M:%S"
name_format = "[%(asctime)s] [%(levelname)s] [%(filename)s:%(lineno)s] - %(message)s"
log = logging.getLogger('')
log.setLevel(logging.DEBUG)
format = logging.Formatter(name_format, datetime.now().strftime(date_format))
console_handler = logging.StreamHandler(sys.stderr)
file_handler = handlers.RotatingFileHandler(filename=datetime.now().strftime(file_name.format(date_format)),
maxBytes=(1048576*5),
backupCount=7)
console_handler.setFormatter(format)
file_handler.setFormatter(format)
log.addHandler(console_handler)
log.addHandler(file_handler)
from time import sleep
log.info("1")
sleep(2)
log.info("2")
sleep(2)
log.info("3")
Here is output:
[2017-07-08_17:20:51] [INFO] [logs.py:112] - 1
[2017-07-08_17:20:51] [INFO] [logs.py:114] - 2
[2017-07-08_17:20:51] [INFO] [logs.py:116] - 3
have a look at the documentation of logging.Formatter(fmt=None, datefmt=None, style='%'). the second argument you need to pass is a datefmt ("%Y-%m-%d_%H:%M:%S" in your case). the logger will do the fmt.strftime(...) for you.
you are passing a string that represents datetime.now() in this format. as this is a str (e.g. '2017-07-08_17:20:51') the formatter does not complain but always prints this exact date: '2017-07-08_17:20:51'.strftime(...) will result in '2017-07-08_17:20:51' - there are no format specifiers to fill in.
what you should do is this:
fmt = logging.Formatter(name_format, date_format)
# instead of
# format = logging.Formatter(name_format, datetime.now().strftime(date_format))
(btw: format is a built-in; renamed your formatter to fmt such that the built-in is not overwritten).
My logger:
logging.basicConfig(filename="{}/log.log".format(config.log_dir), level=logging.DEBUG, format="%(asctime)s %(message)s", datefmt="%d-%b-%Y %I:%M:%S" )
And .. no timestamp is getting added:
logging.info("fdsfdsfds")
# => fdsfdsfds
This works for me
That's my output
import logging
FORMAT = '%(asctime)s %(message)s'
DATE = '%d-%b-%Y %I:%M:%S'
logging.basicConfig(filename="log.log",level=logging.DEBUG,format=FORMAT,datefmt=DATE)
logger = logging.getLogger('Stackoverflow log')
logger.info('Info 1')
If you still get no timestamp please add more code
My current format string is:
formatter = logging.Formatter('%(asctime)s : %(message)s')
and I want to add a new field called app_name which will have a different value in each script that contains this formatter.
import logging
formatter = logging.Formatter('%(asctime)s %(app_name)s : %(message)s')
syslog.setFormatter(formatter)
logger.addHandler(syslog)
But I'm not sure how to pass that app_name value to the logger to interpolate into the format string. I can obviously get it to appear in the log message by passing it each time but this is messy.
I've tried:
logging.info('Log message', app_name='myapp')
logging.info('Log message', {'app_name', 'myapp'})
logging.info('Log message', 'myapp')
but none work.
You could use a LoggerAdapter so you don't have to pass the extra info with every logging call:
import logging
extra = {'app_name':'Super App'}
logger = logging.getLogger(__name__)
syslog = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s %(app_name)s : %(message)s')
syslog.setFormatter(formatter)
logger.setLevel(logging.INFO)
logger.addHandler(syslog)
logger = logging.LoggerAdapter(logger, extra)
logger.info('The sky is so blue')
logs (something like)
2013-07-09 17:39:33,596 Super App : The sky is so blue
Filters can also be used to add contextual information.
import logging
class AppFilter(logging.Filter):
def filter(self, record):
record.app_name = 'Super App'
return True
logger = logging.getLogger(__name__)
logger.addFilter(AppFilter())
syslog = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s %(app_name)s : %(message)s')
syslog.setFormatter(formatter)
logger.setLevel(logging.INFO)
logger.addHandler(syslog)
logger.info('The sky is so blue')
produces a similar log record.
Python3
As of Python3.2 you can now use LogRecordFactory
import logging
logging.basicConfig(format="%(custom_attribute)s - %(message)s")
old_factory = logging.getLogRecordFactory()
def record_factory(*args, **kwargs):
record = old_factory(*args, **kwargs)
record.custom_attribute = "my-attr"
return record
logging.setLogRecordFactory(record_factory)
>>> logging.info("hello")
my-attr - hello
Of course, record_factory can be customized to be any callable and the value of custom_attribute could be updated if you keep a reference to the factory callable.
Why is that better than using Adapters / Filters?
You do not need to pass your logger around the application
It actually works with 3rd party libraries that use their own logger (by just calling logger = logging.getLogger(..)) would now have the same log format. (this is not the case with Filters / Adapters where you need to be using the same logger object)
You can stack/chain multiple factories
You need to pass the dict as a parameter to extra to do it that way.
logging.info('Log message', extra={'app_name': 'myapp'})
Proof:
>>> import logging
>>> logging.basicConfig(format="%(foo)s - %(message)s")
>>> logging.warning('test', extra={'foo': 'bar'})
bar - test
Also, as a note, if you try to log a message without passing the dict, then it will fail.
>>> logging.warning('test')
Traceback (most recent call last):
File "/usr/lib/python2.7/logging/__init__.py", line 846, in emit
msg = self.format(record)
File "/usr/lib/python2.7/logging/__init__.py", line 723, in format
return fmt.format(record)
File "/usr/lib/python2.7/logging/__init__.py", line 467, in format
s = self._fmt % record.__dict__
KeyError: 'foo'
Logged from file <stdin>, line 1
Another way is to create a custom LoggerAdapter. This is particularly useful when you can't change the format OR if your format is shared with code that does not send the unique key (in your case app_name):
class LoggerAdapter(logging.LoggerAdapter):
def __init__(self, logger, prefix):
super(LoggerAdapter, self).__init__(logger, {})
self.prefix = prefix
def process(self, msg, kwargs):
return '[%s] %s' % (self.prefix, msg), kwargs
And in your code, you would create and initialize your logger as usual:
logger = logging.getLogger(__name__)
# Add any custom handlers, formatters for this logger
myHandler = logging.StreamHandler()
myFormatter = logging.Formatter('%(asctime)s %(message)s')
myHandler.setFormatter(myFormatter)
logger.addHandler(myHandler)
logger.setLevel(logging.INFO)
Finally, you would create the wrapper adapter to add a prefix as needed:
logger = LoggerAdapter(logger, 'myapp')
logger.info('The world bores you when you are cool.')
The output will look something like this:
2013-07-09 17:39:33,596 [myapp] The world bores you when you are cool.
I found this SO question after implementing it myself. Hope it helps someone. In the code below, I'm inducing an extra key called claim_id in the logger format. It will log the claim_id whenever there is a claim_id key present in the environment. In my use case, I needed to log this information for an AWS Lambda function.
import logging
import os
LOG_FORMAT = '%(asctime)s %(name)s %(levelname)s %(funcName)s %(lineno)s ClaimID: %(claim_id)s: %(message)s'
class AppLogger(logging.Logger):
# Override all levels similarly - only info overriden here
def info(self, msg, *args, **kwargs):
return super(AppLogger, self).info(msg, extra={"claim_id": os.getenv("claim_id", "")})
def get_logger(name):
""" This function sets log level and log format and then returns the instance of logger"""
logging.setLoggerClass(AppLogger)
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
logger = logging.getLogger(name)
logger.setLevel(logging.INFO)
return logger
LOGGER = get_logger(__name__)
LOGGER.info("Hey")
os.environ["claim_id"] = "12334"
LOGGER.info("Hey")
Gist: https://gist.github.com/ygivenx/306f2e4e1506f302504fb67abef50652
If you need a default extra mapping, and you want to customize it for ad-hoc log messages, this works in Python 2.7+ by creating a LoggerAdapter that merges a default extra dictionary with any extra from a given message.
import logging
import os
import sys
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(levelname)-8s Py%(python)-4s pid:%(pid)-5s %(message)s',
)
_logger = logging.getLogger("my-logger")
_logger.setLevel(logging.DEBUG)
class DefaultExtrasAdapter(logging.LoggerAdapter):
def __init__(self, logger, extra):
super(DefaultExtrasAdapter, self).__init__(logger, extra)
def process(self, msg, kwargs):
# Speed gain if no extras are present
if "extra" in kwargs:
copy = dict(self.extra).copy()
copy.update(kwargs["extra"])
kwargs["extra"] = copy
else:
kwargs["extra"] = self.extra
return msg, kwargs
LOG = DefaultExtrasAdapter(_logger, {"python": sys.version_info[0], "pid": os.getpid()})
if __name__ == "__main__":
LOG.info("<-- With defaults")
LOG.info("<-- With my version", extra={"python": 3.10})
LOG.info("<-- With my pid", extra={"pid": 0})
LOG.info("<-- With both", extra={"python": 2.7, "pid": -1})
Results:
2021-08-05 18:58:27,308 INFO Py2 pid:8435 <-- With defaults
2021-08-05 18:58:27,309 INFO Py3.1 pid:8435 <-- With my version
2021-08-05 18:58:27,309 INFO Py2 pid:0 <-- With my pid
2021-08-05 18:58:27,309 INFO Py2.7 pid:-1 <-- With both
The accepted answer did not log the format in logfile, whereas the format was reflected in sys output.
Alternatively I used a simpler approach and worked as;
logging.basicConfig(filename="mylogfile.test",
filemode="w+",
format='%(asctime)s: ' +app_name+': %(message)s ',
level=logging.DEBUG)
Using mr2ert's answer, I came up with this comfortable solution (Though I guess it's not recommended) - Override the built-in logging methods to accept the custom argument and create the extra dictionary inside the methods:
import logging
class CustomLogger(logging.Logger):
def debug(self, msg, foo, *args, **kwargs):
extra = {'foo': foo}
if self.isEnabledFor(logging.DEBUG):
self._log(logging.DEBUG, msg, args, extra=extra, **kwargs)
*repeat for info, warning, etc*
logger = CustomLogger('CustomLogger', logging.DEBUG)
formatter = logging.Formatter('%(asctime)s [%(foo)s] %(message)s')
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.debug('test', 'bar')
Output:
2019-03-02 20:06:51,998 [bar] test
This is the built in function for reference:
def debug(self, msg, *args, **kwargs):
"""
Log 'msg % args' with severity 'DEBUG'.
To pass exception information, use the keyword argument exc_info with
a true value, e.g.
logger.debug("Houston, we have a %s", "thorny problem", exc_info=1)
"""
if self.isEnabledFor(DEBUG):
self._log(DEBUG, msg, args, **kwargs)
import logging;
class LogFilter(logging.Filter):
def __init__(self, code):
self.code = code
def filter(self, record):
record.app_code = self.code
return True
logging.basicConfig(format='[%(asctime)s:%(levelname)s]::[%(module)s -> %(name)s] - APP_CODE:%(app_code)s - MSG:%(message)s');
class Logger:
def __init__(self, className):
self.logger = logging.getLogger(className)
self.logger.setLevel(logging.ERROR)
#staticmethod
def getLogger(className):
return Logger(className)
def logMessage(self, level, code, msg):
self.logger.addFilter(LogFilter(code))
if level == 'WARN':
self.logger.warning(msg)
elif level == 'ERROR':
self.logger.error(msg)
else:
self.logger.info(msg)
class Test:
logger = Logger.getLogger('Test')
if __name__=='__main__':
logger.logMessage('ERROR','123','This is an error')
I am trying to setup a format for logging in python:
import logging,logging.handlers
FORMAT = "%(asctime)-15s %(message)s"
logging.basicConfig(format=FORMAT,level=logging.INFO)
logger = logging.getLogger("twitter")
handler = logging.handlers.RotatingFileHandler('/var/log/twitter_search/message.log', maxBytes=1024000, backupCount=5)
logger.addHandler(handler)
Basically, logging works, but without the date format...
You can add the datefmt parameter to basicConfig:
logging.basicConfig(format=FORMAT,level=logging.INFO,datefmt='%Y-%m-%d %H:%M:%S')
Or, to set the format for the Rotating FileHandler:
fmt = logging.Formatter(FORMAT,datefmt='%Y-%m-%d')
handler.setFormatter(fmt)
Basic example:
import logging
logging.basicConfig(
format='%(asctime)s %(levelname)s %(message)s',
level=logging.INFO,
datefmt='%Y-%m-%d %H:%M:%S'
)
logging.info('Just a random string...')
# 2030-01-01 00:00:00 INFO Just a random string...
If you want to adjust the line spacing between levelname and message change the %(levelname)s like the example:
... %(levelname)-10s ...
# 2030-01-01 00:00:00 INFO Just a random string...