So I build a small logger class that logs everything but there is a small issue with it.
Then some kind of exception/error happens outside of try... except block I don't have a way to log it into a file.
So the question is, how to catch errors out side of try... except block.
Also if there someone saying "Put whole program in try except.", well here is the problem that it completely ignores errors happening in threads and subprocesses.
Example:
from datetime import datetime
import logging
class Msg:
def __init__(self, logger_name, path, conn=None, info_format="utf-8", file_name=None):
self.path = path
self.logger = logging.getLogger(logger_name)
self.logger.setLevel(logging.DEBUG)
self.now = datetime.now()
self.now = self.now.strftime("%d_%m_%Y-%H_%M_%S")
self.file_formatter = logging.Formatter("%(asctime)s:%(levelname)s:%(message)s")
if file_name is not None:
self.now = file_name
self.file_handler = logging.FileHandler(f"{self.path}/{self.now}.log")
self.file_handler.setLevel(logging.DEBUG)
self.file_handler.setFormatter(self.file_formatter)
self.stream_formatter = logging.Formatter("%(levelname)s:%(message)s")
self.stream_handler = logging.StreamHandler()
self.stream_handler.setFormatter(self.stream_formatter)
self.logger.addHandler(self.stream_handler)
self.logger.addHandler(self.file_handler)
def debug(self, message):
self.logger.debug(message)
def info(self, message):
self.logger.info(message)
def warning(self, message):
self.logger.warning(message)
def error(self, message): # Somehow any exception outside try except have to trigger this to be logged or any other function.
self.logger.error(message, exc_info=True)
def critical(self, message):
self.logger.critical(message, exc_info=True)
if __name__ == "__main__":
log = Msg("Example_Logger_Name", ".")|
a = 1/0 # an example error
Related
I'm using PTB library and have many handlers as:
def some_handler(update, context): # Update is new data from user, context is all my data
do_something()
And I want to notify user if error has occured, like:
def some_handler(update, context):
try:
do_something()
except Exception as e:
notify_user(text="Some error occured")
logger.error(e)
To follow DRY and make code more nicely I wrote such decorator:
def bot_logger(text: str = ''):
def decorator(function):
#loguru_logger.catch
#wraps(function)
def wrapper(*args, **kwargs):
try:
return function(*args, **kwargs)
except Exception as e:
notify_user(text=f'Unknown error. {text} Please try againt latter')
loguru_logger.error(e
return wrapper
return decorator
But in most cases I'm getting a pretty obscure log record like
2021-11-26 19:47:32.558 | ERROR | bot:wrapper:46 - There are no messages to send
Question:
How I can make this message a more informative as a standard python error?
What should I fix in bot_logger decorator?
Logger setup:
from loguru import logger as loguru_logger
loguru_logger.add(
sink="log_error.txt",
filter=lambda record: record["level"].name == "ERROR",
backtrace=True,
format="{time} {level} {function}:{line} {message}",
level="ERROR",
rotation="1 MB",
compression="zip",
enqueue=True,
diagnose=True
)
P.S. I checked another similar questions
Best practices for logging in python
Python logging using a decorator
And the others but don't found an asnwer
Also, I tried different logger formats and parameters but it's not change a log record a much
I'm new to python and trying to create wrapper over logging to reuse changes needed to modify formatting etc.
I've written my wrapper class in following way -
import logging
import sys
from datetime import datetime
class CustomLogger:
"""This is custom logger class"""
_format_spec = f"[%(name)-24s | %(asctime)s | %(levelname)s ] (%(filename)-32s : %(lineno)-4d) ==> %(message)s"
_date_format_spec = f"%Y-%m-%d # %I:%M:%S %p"
def __init__(self, name, level=logging.DEBUG, format_spec=None):
""""""
self.name = name
self.level = level
self.format_spec = format_spec if format_spec else CustomLogger._format_spec
# Complete logging configuration.
self.logger = self.get_logger(self.name, self.level)
def get_file_handler(self, name, level):
"""This is a method to get a file handler"""
today = datetime.now().strftime(format="%Y-%m-%d")
file_handler = logging.FileHandler("{}-{}.log".format(name, today))
file_handler.setLevel(level)
file_handler.setFormatter(logging.Formatter(self.format_spec,
datefmt=CustomLogger._date_format_spec))
return file_handler
def get_stream_handler(self, level):
"""This is a method to get a stream handler"""
stream_handler = logging.StreamHandler(sys.stdout)
stream_handler.setLevel(level)
stream_handler.setFormatter(logging.Formatter(self.format_spec,
datefmt=CustomLogger._date_format_spec))
return stream_handler
def get_logger(self, name, level):
"""This is a method to get a logger"""
logger = logging.getLogger(name)
logger.addHandler(self.get_file_handler(name, level))
# logger.addHandler(self.get_stream_handler(level))
return logger
def info(self, msg):
"""info message logger method"""
self.logger.info(msg)
def error(self, msg):
"""error message logger method"""
self.logger.error(msg)
def debug(self, msg):
"""debug message logger method"""
self.logger.debug(msg)
def warn(self, msg):
"""warning message logger method"""
self.logger.warn(msg)
def critical(self, msg):
"""critical message logger method"""
self.logger.critical(msg)
def exception(self, msg):
"""exception message logger method"""
self.logger.exception(msg)
But when I try to use my CustomLogger, nothing goes into the log file.
def main():
"""This main function"""
logger = CustomLogger(name="debug", level=logging.DEBUG)
logger.info("Called main")
if __name__ == "__main__":
main()
If I do similar thing without class/function wrapper, it works. Not sure where I'm going wrong. Any pointer will help.
Further update on the question
After making this (custom_logger.py) as a package and using in actual application (app.py), I'm noticing it always prints custom_logger.py as filename but not app.py.
How to fix this? I'm ok with rewriting the CustomLogger class if required.
I missed to do setLevel() for the logger. After doing that, problem is resolved. I also added pid for the file handler file-name to avoid any future issue with multi-process env.
Let me know if there's anything I can do better here wrt any other potential issues.
I have a class in which I log errors and raise them. However, I have different functions which expect that error.
Now, even if these functions except the error properly, it is still logged. This leads to confusing log files in which multiple conflicting entries can be seen. For example:
import logging
logging.basicConfig(filename = "./log")
logger = logging.getLogger()
logger.setLevel(logging.INFO)
class Foo:
def __init__(self):
pass
def foo_error(self):
logger.error("You have done something very stupid!")
raise RuntimeError("You have done something very stupid!")
def foo_except(self):
try:
self.foo_error()
except RuntimeError as error:
logger.info("It was not so stupid after all!")
Foo = Foo()
Foo.foo_except()
Here, both messages show up in "./log". Preferably, I would like to suppress the first error log message if it is caught later on.
I have not seen an answer anywhere else. Maybe the way I am doing things suggests bad design. Any ideas?
You cannot really ask Python whether an exception will be caught lateron. So your only choice is to log only after you know whether the exception was caught or not.
One possible solution (though I'm not sure if this will work in your context):
import logging
logging.basicConfig(filename = "./log")
logger = logging.getLogger()
class Foo:
def __init__(self):
pass
def foo_error(self):
# logger.error("You have done something very stupid!")
raise RuntimeError("You have done something very stupid!")
def foo_except(self):
try:
self.foo_error()
except RuntimeError as error:
logger.warning("It was not so stupid after all!")
try:
Foo = Foo()
Foo.foo_except()
Foo.foo_error()
except Exception as exc:
if isinstance(exc, RuntimeError):
logger.error("%s", exc)
raise
After some more thinking and several failed attempts, i arrived at the following answer.
Firstly, as #gelonida mentioned:
You cannot really ask Python whether an exception will be caught later on.
This implies that a log entry which also raises an exception has to be written, because if the exception is not caught later on, the log entry is wiped out and missing from the file.
So instead of trying to control which log message gets written to file, we should implement a way to delete voided log messages from the file.
import logging
logging.basicConfig(filename = "./log")
logger = logging.getLogger()
logger.setLevel(logging.INFO)
class Foo:
def __init__(self):
pass
def foo_error(self):
logger.error("You have done something very stupid!")
raise RuntimeError("You have done something very stupid!")
def foo_except(self):
try:
self.foo_error()
except RuntimeError as error:
logger.info("It was not so stupid after all!")
Foo = Foo()
Foo.foo_except()
Following that logic we should replace in the original example above the line logger.info("It was not so stupid after all!") with a function that deletes the last committed log message and logs the correct one instead!
One way to achieve this is to modify the logging class and add two components. Namely a log record history and a FileHandler which supports deletion of log records. Let's start with the log record history.
class RecordHistory:
def __init__(self):
self._record_history = []
def write(self, record):
self._record_history.append(record)
def flush(self):
pass
def get(self):
return self._record_history[-1]
def pop(self):
return self._record_history.pop()
This is basically a data container which implements the write and flush methods alongside some other conveniences. The write and flush methods are required by logging.StreamHandler. For more information visit the logging.handlers documentation.
Next, we modify the existing logging.FileHandler to support the revoke method. This method allows us to delete a specific log record.
import re
class RevokableFileHandler(logging.FileHandler):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def revoke(self, record):
with open(self.baseFilename, mode="r+") as log:
substitute = re.sub(rf"{record}", "", log.read(), count=1)
log.seek(0)
log.write(substitute)
log.truncate()
Finally, we modify the Logger class. Note however that we cannot inherit from logging.Logger directly as stated here. Additionally, we add a logging.StreamHandler which pushes log records to our RecordHistory object. Also we implement a addRevokableHandler method which registers all handlers that support revoking records.
import logging
class Logger(logging.getLoggerClass()):
def __init__(self, name):
super().__init__(name)
self.revokable_handlers = []
self.record_history = RecordHistory()
stream_handler = logging.StreamHandler(stream=self.record_history)
stream_handler.setLevel(logging.INFO)
self.addHandler(stream_handler)
def addRevokableHandler(self, handler):
self.revokable_handlers.append(handler)
super().addHandler(handler)
def pop_and_log(self, level, msg):
record = self.record_history.pop()
for handler in self.revokable_handlers:
handler.revoke(record)
self.log(level, msg)
This leads to the following solution in the original code:
logging.setLoggerClass(Logger)
logger = logging.getLogger("root")
logger.setLevel(logging.INFO)
file_handler = RevokableFileHandler("./log")
file_handler.setLevel(logging.INFO)
logger.addRevokableHandler(file_handler)
class Foo:
def __init__(self):
pass
def foo_error(self):
logger.error("You have done something very stupid!")
raise RuntimeError("You have done something very stupid!")
def foo_except(self):
try:
self.foo_error()
except KeyError as error:
logger.pop_and_log(logging.INFO, "It was not so stupid after all!")
Foo = Foo()
Foo.foo_except()
Hopefully this lengthy answer can be of use to someone. Though it is still not clear to me if logging errors and info messages in such a way is considered bad code design.
I have a program that exposes different API, some of them require a huge amount of time.
Sometimes it happens that an user insert wrong data in the algorithm and has to abort the execution.
I'm trying to create that system raising a keyboardException and catching from another API, but actually without any result.
Can someone please address me ? Below my test code
#ns.route('/abort')
class ProjectItem(Resource):
def get(self):
raise KeyboardInterrupt
return None,200
#ns.route('/test')
class ProjectItem(Resource):
def get(self):
try:
print('start')
time.sleep(5)
print('end')
return None,400
except KeyboardInterrupt:
print("interrupted, yay!")
return None,200
I have even tryed this way
#ns.route('/abort')
class ProjectItem(Resource):
def get(self):
print('abort')
sys.exit("pls pls pls")
print('aborting')
return None,200
#ns.route('/test')
class ProjectItem(Resource):
def get(self):
try:
print('start')
time.sleep(5)
print('end')
return None,400
except:
print("interrupted, yay!")
return None,200
but still unable to see that agognated "yay"
I have a script executing several independent functions in turn. I would like to collect the errors/exceptions happening along the way, in order to send an email with a summary of the errors.
What is the best way to raise these errors/exceptions and collect them, while allowing the script to complete and go through all the steps? They are independent, so it does not matter if one crashes. The remaining ones can still run.
def step_1():
# Code that can raise errors/exceptions
def step_2():
# Code that can raise errors/exceptions
def step_3():
# Code that can raise errors/exceptions
def main():
step_1()
step_2()
step_3()
send_email_with_collected_errors()
if '__name__' == '__main__':
main()
Should I wrap each step in a try..except block in the main() function? Should I use a decorator on each step function, in addition to an error collector?
You could wrap each function in try/except, usually better for small simple scripts.
def step_1():
# Code that can raise errors/exceptions
def step_2():
# Code that can raise errors/exceptions
def step_3():
# Code that can raise errors/exceptions
def main():
try:
step_1_result = step_1()
log.info('Result of step_1 was {}'.format(result))
except Exception as e:
log.error('Exception raised. {}'.format(e))
step_1_result = e
continue
try:
step_2_result = step_2()
log.info('Result of step_2 was {}'.format(result))
except Exception as e:
log.error('Exception raised. {}'.format(e))
step_2_result = e
continue
try:
step_3_result = step_3()
log.info('Result of step_3 was {}'.format(result))
except Exception as e:
log.error('Exception raised. {}'.format(e))
step_3_result = e
continue
send_email_with_collected_errors(
step_1_result,
step_2_result,
step_3_result
)
if '__name__' == '__main__':
main()
For something more elaborate you could use a decorator that'd construct a list of errors/exceptions caught. For example
class ErrorIgnore(object):
def __init__(self, errors, errorreturn=None, errorcall=None):
self.errors = errors
self.errorreturn = errorreturn
self.errorcall = errorcall
def __call__(self, function):
def returnfunction(*args, **kwargs):
try:
return function(*args, **kwargs)
except Exception as E:
if type(E) not in self.errors:
raise E
if self.errorcall is not None:
self.errorcall(E, *args, **kwargs)
return self.errorreturn
return returnfunction
Then you could use it like this:
exceptions = []
def errorcall(E, *args):
print 'Exception raised {}'.format(E)
exceptions.append(E)
#ErrorIgnore(errors=[ZeroDivisionError, ValueError], errorreturn=None, errorcall=errorcall)
def step_1():
# Code that can raise errors/exceptions
...
def main():
step_1()
step_2()
step_3()
send_email_with_collected_errors(exceptions)
if '__name__' == '__main__':
main()
use simple try except statements and do logging for the exceptions that would be standard way to collect all your errors.
There are two options:
Use decorator in which you catch all exceptions and save it somewhere.
Add try/except everywhere.
Using decorator might be much better and cleaner, and code will be easier to maintain.
How to store errors? Your decision. You can add them to some list, create logging class receiving exceptions and get them after everything is done… Depends on your project and size of code.
Simple logging class:
class LoggingClass(object):
def __init__(self):
self.exceptions = []
def add_exception(self, exception):
self.exceptions.append(exception)
def get_all(self):
return self.exceptions
Create instance of class in your script, catch exceptions in decorator and add them to class (however global variable might be also ok).