I'm new to python and trying to create wrapper over logging to reuse changes needed to modify formatting etc.
I've written my wrapper class in following way -
import logging
import sys
from datetime import datetime
class CustomLogger:
"""This is custom logger class"""
_format_spec = f"[%(name)-24s | %(asctime)s | %(levelname)s ] (%(filename)-32s : %(lineno)-4d) ==> %(message)s"
_date_format_spec = f"%Y-%m-%d # %I:%M:%S %p"
def __init__(self, name, level=logging.DEBUG, format_spec=None):
""""""
self.name = name
self.level = level
self.format_spec = format_spec if format_spec else CustomLogger._format_spec
# Complete logging configuration.
self.logger = self.get_logger(self.name, self.level)
def get_file_handler(self, name, level):
"""This is a method to get a file handler"""
today = datetime.now().strftime(format="%Y-%m-%d")
file_handler = logging.FileHandler("{}-{}.log".format(name, today))
file_handler.setLevel(level)
file_handler.setFormatter(logging.Formatter(self.format_spec,
datefmt=CustomLogger._date_format_spec))
return file_handler
def get_stream_handler(self, level):
"""This is a method to get a stream handler"""
stream_handler = logging.StreamHandler(sys.stdout)
stream_handler.setLevel(level)
stream_handler.setFormatter(logging.Formatter(self.format_spec,
datefmt=CustomLogger._date_format_spec))
return stream_handler
def get_logger(self, name, level):
"""This is a method to get a logger"""
logger = logging.getLogger(name)
logger.addHandler(self.get_file_handler(name, level))
# logger.addHandler(self.get_stream_handler(level))
return logger
def info(self, msg):
"""info message logger method"""
self.logger.info(msg)
def error(self, msg):
"""error message logger method"""
self.logger.error(msg)
def debug(self, msg):
"""debug message logger method"""
self.logger.debug(msg)
def warn(self, msg):
"""warning message logger method"""
self.logger.warn(msg)
def critical(self, msg):
"""critical message logger method"""
self.logger.critical(msg)
def exception(self, msg):
"""exception message logger method"""
self.logger.exception(msg)
But when I try to use my CustomLogger, nothing goes into the log file.
def main():
"""This main function"""
logger = CustomLogger(name="debug", level=logging.DEBUG)
logger.info("Called main")
if __name__ == "__main__":
main()
If I do similar thing without class/function wrapper, it works. Not sure where I'm going wrong. Any pointer will help.
Further update on the question
After making this (custom_logger.py) as a package and using in actual application (app.py), I'm noticing it always prints custom_logger.py as filename but not app.py.
How to fix this? I'm ok with rewriting the CustomLogger class if required.
I missed to do setLevel() for the logger. After doing that, problem is resolved. I also added pid for the file handler file-name to avoid any future issue with multi-process env.
Let me know if there's anything I can do better here wrt any other potential issues.
Related
So I build a small logger class that logs everything but there is a small issue with it.
Then some kind of exception/error happens outside of try... except block I don't have a way to log it into a file.
So the question is, how to catch errors out side of try... except block.
Also if there someone saying "Put whole program in try except.", well here is the problem that it completely ignores errors happening in threads and subprocesses.
Example:
from datetime import datetime
import logging
class Msg:
def __init__(self, logger_name, path, conn=None, info_format="utf-8", file_name=None):
self.path = path
self.logger = logging.getLogger(logger_name)
self.logger.setLevel(logging.DEBUG)
self.now = datetime.now()
self.now = self.now.strftime("%d_%m_%Y-%H_%M_%S")
self.file_formatter = logging.Formatter("%(asctime)s:%(levelname)s:%(message)s")
if file_name is not None:
self.now = file_name
self.file_handler = logging.FileHandler(f"{self.path}/{self.now}.log")
self.file_handler.setLevel(logging.DEBUG)
self.file_handler.setFormatter(self.file_formatter)
self.stream_formatter = logging.Formatter("%(levelname)s:%(message)s")
self.stream_handler = logging.StreamHandler()
self.stream_handler.setFormatter(self.stream_formatter)
self.logger.addHandler(self.stream_handler)
self.logger.addHandler(self.file_handler)
def debug(self, message):
self.logger.debug(message)
def info(self, message):
self.logger.info(message)
def warning(self, message):
self.logger.warning(message)
def error(self, message): # Somehow any exception outside try except have to trigger this to be logged or any other function.
self.logger.error(message, exc_info=True)
def critical(self, message):
self.logger.critical(message, exc_info=True)
if __name__ == "__main__":
log = Msg("Example_Logger_Name", ".")|
a = 1/0 # an example error
Is there an easy way to execute a python script and then have all error messages save to a type of log file (csv, txt, anything).
class MyClass():
def __init__(self, something):
self.something = something
def my_function(self):
# code here
Or is the only way to add try and except statements everywhere and write the error messages to a file?
Yes you can do that using python logging
Here's a specific example, using the info from https://realpython.com/python-logging/ with your code:
import logging
logging.basicConfig(filename='app.log', filemode='w', format='%(name)s - %(levelname)s - %(message)s')
logging.warning('This will get logged to a file')
class MyClass():
def __init__(self, something):
self.something = something
def my_function(self):
logging.warning('my_function entered...')
After you instantiate your class and call my_function, you should get logging output in your log file (here app.log):
root - WARNING - This will get logged to a file
root - WARNING - my_function entered...
I´m just starting to learn Python and have encountered a Problem that I´m not able to solve.
I want to redirect every level above CRITICAL to sys.stderr and everything above WARNING to sys.stdout.
I came up with this script...
import logging
import sys
print("imported module {}".format(__name__))
class PyLogger(logging.Logger):
"""Wrapper for logging.Logger to redirect its message to
sys.stdout or sys.stderr accordingly """
def __init__(self, *args):
super(PyLogger, self).__init__(self, *args)
# get Logger
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
# build Formatter
formatter = logging.Formatter(fmt="%(asctime)s:%(name)s %(message)s")
# build StreamHandler for sys.stderr
error = logging.StreamHandler(stream=sys.stderr)
error.setLevel(logging.CRITICAL)
error.setFormatter(formatter)
logger.addHandler(error)
# build StreamHandler for sys.stdin
out = logging.StreamHandler(stream=sys.stdout)
out.setFormatter(formatter)
out.setLevel(logging.WARNING)
logger.addHandler(out)
def main():
logger = PyLogger()
# help(logger)
logger.info("INFO")
if __name__ == "__main__":
main()
When running this scrip directly I get the following error:
No handlers could be found for logger "<__main__.PyLogger object at 0x105f23c50>"
I´ve googled around and many people said that a logging.basicConfig() would do the job but that didn´t worked for me.
Maybe someone of you guys could help me out.
Thanks!
Your class subclasses logging.Logger, so you should not call getLogger or manipulate a logger as an attribute. Rather, the logger is self inside the class, and should be adjusted directly:
import logging
import sys
print("imported module {}".format(__name__))
class PyLogger(logging.Logger):
"""Wrapper for logging.Logger to redirect its message to
sys.stdout or sys.stderr accordingly """
def __init__(self, *args):
super(PyLogger, self).__init__(self, *args)
#####
# self *is* the logger!
self.setLevel(logging.DEBUG)
# build Formatter
formatter = logging.Formatter(fmt="%(asctime)s:%(name)s %(message)s")
# build StreamHandler for sys.stderr
error = logging.StreamHandler(stream=sys.stderr)
error.setLevel(logging.CRITICAL)
error.setFormatter(formatter)
#####
# Assign the handler to self
self.addHandler(error)
# build StreamHandler for sys.stdin
out = logging.StreamHandler(stream=sys.stdout)
out.setFormatter(formatter)
out.setLevel(logging.WARNING)
#####
# Assign the handler to self
self.addHandler(out)
def main():
logger = PyLogger()
# help(logger)
logger.info("INFO")
logger.warning("WARN")
logger.critical("CRIT")
if __name__ == "__main__":
main()
This displays the following, as expected:
ely#eschaton:~/programming$ python test_logger.py
imported module __main__
2018-03-01 11:59:41,896:<__main__.PyLogger object at 0x7fa236aa4a50> WARN
2018-03-01 11:59:41,896:<__main__.PyLogger object at 0x7fa236aa4a50> CRIT
2018-03-01 11:59:41,896:<__main__.PyLogger object at 0x7fa236aa4a50> CRIT
Notice how the critical message trips two different output handlers, so it appears twice (once because it satisfied warning level, once for critical level).
In your original code, notice that you are creating a variable called logger inside of __init__, but this not assigned to self or anything. This variable gets destroyed when it goes out of scope of the __init__ function, and so the assignment of any handlers is meaningless. Plus, because handlers weren't assigned to self, but the object that self is referencing is the logger that will be called later on, that is why you see the error about no handlers.
I have the following class:
class Log(object):
# class new
#new is used instead of init because __new__ is able to return (where __init__ can't)
def __new__(self, name, consolelevel, filelevel):
formatter = logging.Formatter('%(asctime)s %(levelname)s: %(name)s: %(message)s')
#Create consolehandler and set formatting (level is set in the ROOT)
consolehandler = StreamHandler()
consolehandler.setFormatter(formatter)
#Create filehandler, set level and set formatting
filehandler = FileHandler(name + '.log')
filehandler.setLevel(filelevel)
filehandler.setFormatter(formatter)
#Create the root logger, add console and file logger. Set the rootlevel == consolelevel.
self.root = logging.getLogger(name)
#causing me problems....
self.root.setLevel(consolelevel)
self.root.addHandler(consolehandler)
self.root.addHandler(filehandler)
self.root.propagate = True
return self.root
# Close the logger object
def close():
# to be implemented
pass
I use this class to log to the console and to a file (depending on the set level). The problem is that the root level seems to be leading for the addhandlers. Is there a way to disable this? Now I set the rootlevel to the same level as the consolelevel but this does not work...
Any advice?
Thanks in advance and with best regards,
JR
A problem that I can see in your code is that it will add more handlers whenever you instantiate the Log class. You probably do not want this.
Keep in mind that getLogger returns always the same instance when called with the same argument, and basically it implements the singleton pattern.
Hence when you later call addHandler it will add a new handler everytime.
The way to deal with logging is to create a logger at the module level and use it.
Also I'd avoid using __new__. In your case you can use a simple function. And note that your Log.close method wont work, because your __new__ method does not return a Log instance, and thus the returned logger doesn't have that method.
Regarding the level of the logger, I don't understand why you do not set the level on the consolehandler but on the whole logger.
This is a simplified version of the module I am making. The module contains a few classes that all need a logging functionality. Each class logs to a different file and and it should also be possible to change the file handler levels between classes (e.g. gamepad class: console.debug and filehandler.info and MQTT class: console info and filehandler.debug).
Therefor I thought that setting up a log class would be the easiest way. Please bear in mind that I usually do electronics but now combined with python. So my skills are pretty basic....
#!/bin/env python2.7
from future import division
from operator import *
import logging
from logging import FileHandler
from logging import StreamHandler
import pygame
import threading
from pygame.locals import *
import mosquitto
import time
from time import sleep
import sys
class ConsoleFileLogger(object):
# class constructor
def __init__(self, filename, loggername, rootlevel, consolelevel, filelevel):
# logger levels: DEBUG, INFO, WARNING, ERROR, CRITICAL
# define a root logger and set its ROOT logging level
logger = logging.getLogger(loggername)
logger.setLevel(rootlevel)
# define a Handler which writes messages or higher to the sys.stderr (Console)
self.console = logging.StreamHandler()
# set the logging level
self.console.setLevel(consolelevel)
# define a Handler which writes messages to a logfile
self.logfile = logging.FileHandler(filename + '.log')
# set the logging level
self.logfile.setLevel(filelevel)
# create formatter and add it to the handlers
formatter = logging.Formatter('%(asctime)s %(levelname)s: %(name)s: %(message)s')
self.console.setFormatter(formatter)
self.logfile.setFormatter(formatter)
# add the handlers to the root logger
logger.addHandler(self.console)
logger.addHandler(self.logfile)
self._logger = logger
# set a net instance of the logger
def set(self):
return self._logger
# Stop and remove the ConsoleFileLogger object
def remove(self):
self._logger.removeHandler(self.console)
self._logger.removeHandler(self.logfile)
self._logfile.FileHandler().close()
class Gamepad():
# class constructor
def __init__(self, mqttgamepad):
self.logger = ConsoleFileLogger('BaseLogFiles/Gamepad', 'Gamepad', logging.INFO, logging.INFO, logging.INFO).set()
if joystickcount == 0:
self.logger.error('No gamepad connected')
elif joystickcount == 1:
self.gamepad = pygame.joystick.Joystick(0)
self.gamepad.init()
self.logger.debug('Joystick name %s', self.gamepad.get_name())
self.logger.debug('nb of axes = %s', self.gamepad.get_numaxes())
self.logger.debug('nb of balls = %s', self.gamepad.get_numballs())
self.logger.debug('nb of buttons = %s', self.gamepad.get_numbuttons())
self.logger.debug('nb of mini joysticks = %s', self.gamepad.get_numhats())
elif joystickcount > 1:
self.logger.error('only one gamepad is allowed')
def run(self):
self.logger.debug('gamepad running')
class MQTTClient():
def __init__(self, clientname):
self.logger = ConsoleFileLogger('BaseLogFiles/MQTT/Pub', clientname, logging.DEBUG, logging.DEBUG, logging.DEBUG).set()
self.logger.debug('test')
def run(self):
self.logger.info('Connection MQTT Sub OK')
def main():
logger = ConsoleFileLogger('BaseLogFiles/logControlCenterMain', 'ControlCenterMain', logging.DEBUG, logging.DEBUG, logging.DEBUG).set()
mqttclient = MQTTClient("MQTTClient")
mqttclient.connect()
gamepad = Gamepad(mqttclient)
if gamepad.initialized():
gamepadthread = threading.Thread(target=gamepad.run)
gamepadthread.start()
mqtttpubhread = threading.Thread(target=mqttclient.run)
mqtttpubhread.start()
logger.info('BaseMain started')
# Monitor the running program for a KeyboardInterrupt exception
# If this is the case all threads and other methods can be closed the correct way :)
while 1:
try:
sleep(1)
except KeyboardInterrupt:
logger.info('Ctrl-C pressed')
gamepad.stop()
mqttclient.stop()
logger.info('BaseMain stopped')
sys.exit(0)
if name == 'main':
main()
I have a python test in which I want to test if the logging works properly. For example I have a function that creates a user and at the end the logging writes to log file the response.
logger = logging.getLogger('mylogger')
logger.setLevel(logging.DEBUG)
handler = logging.handlers.WatchedFileHandler('mylogfile.log')
formatter = logging.Formatter('%(asctime)s: %(message)s',
'%d/%b/%Y:%H:%M:%S %z')
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.info('Some log text')
In my test case I want to send the log output to the StringIO.
class MyTest(unittest.TestCase):
def setUp(self):
stream = StringIO()
self.handler = logging.StreamHandler(stream)
log = logging.getLogger('mylogger')
log.removeHandler(log.handlers[0])
log.addHandler(self.handler)
def tearDown(self):
log = logging.getLogger('mylogger')
log.removeHandler(self.handler)
self.handler.close()
The problem is that I'm not sure how should I test wheter or not my logger is working.
Here is an example that works, make sure you set the level of your log, and flush the buffer.
class MyTest(unittest.TestCase):
def setUp(self):
self.stream = StringIO()
self.handler = logging.StreamHandler(self.stream)
self.log = logging.getLogger('mylogger')
self.log.setLevel(logging.INFO)
for handler in self.log.handlers:
self.log.removeHandler(handler)
self.log.addHandler(self.handler)
def testLog(self):
self.log.info("test")
self.handler.flush()
print '[', self.stream.getvalue(), ']'
self.assertEqual(self.stream.getvalue(), 'test')
def tearDown(self):
self.log.removeHandler(self.handler)
self.handler.close()
if __name__=='__main__':
unittest.main()
Further reading, here is an example Temporily Capturing Python Logging to a string buffer that show how you could also do formatting.