Related
I'm trying to use the string.format() style with python logging module. I copied some examples from this site and modified it with the the new formatter. I'd like to use logging.config.dictConfig to specify the logging format.
My script is:
import sys, logging, logging.config
DEFAULT_LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'standard': {
'format': '{message}',
'style': '{' # use string.format()
},
},
'handlers': {
'default': {
'level': 'INFO',
'formatter': 'standard',
'class': 'logging.StreamHandler',
'stream': 'ext://sys.stdout', # Default is stderr
},
},
'loggers': {
'__main__': { # if __name__ == '__main__'
'handlers': ['default'],
'level': 'DEBUG',
'formatter': 'standard'
},
'': {
'handlers': ['default'],
'level': 'INFO',
'formatter': 'standard'
},
}
}
logger = logging.getLogger(__name__)
logging.config.dictConfig(DEFAULT_LOGGING)
if __name__ == '__main__':
logger.info('Hello, {}', 'log')
sys.exit(0)
But an exception raised:
--- Logging error --- Traceback (most recent call last): File "/usr/lib/python3.8/logging/init.py", line 1081, in emit
msg = self.format(record) File "/usr/lib/python3.8/logging/init.py", line 925, in format
return fmt.format(record) File "/usr/lib/python3.8/logging/init.py", line 664, in format
record.message = record.getMessage() File "/usr/lib/python3.8/logging/init.py", line 369, in getMessage
msg = msg % self.args TypeError: not all arguments converted during string formatting Call stack: File "test/dbg/log.py", line
48, in
logger.info('Hello, {}', 'log') Message: 'Hello, {}' Arguments: ('log',)
For some reason python is trying to format using % formatter (msg = msg % self.args). How can I modify my script to point to the correct formatting? Any help will be appreciated.
Thanks in advance.
-Uri
One way of addressing this issue is to create a thin adapter around the built-in logging formatter(s).
Example:
import sys, logging, logging.config
class standardFormatterFactory(logging.Formatter):
def __init__(self, fmt, datefmt, style, validate=False):
try:
# 3.x
super(standardFormatterFactory, self).__init__(fmt=fmt, datefmt=datefmt, style=style, validate=validate)
except:
# 2.7
super(standardFormatterFactory, self).__init__(fmt=fmt, datefmt=datefmt)
self._usesTime = self._fmt.find('%(asctime)') >= 0 or self._fmt.find('{asctime}') >= 0
def usesTime(self):
'''Override logging.Formatter.usesTime()'''
return self._usesTime
def format(self, record):
'''Override logging.Formatter.format()'''
try:
return super(standardFormatterFactory, self).format(record=record)
except TypeError as ex:
# message contains {} formate specifiers
record.message = record.msg.format(*record.args)
if self.usesTime():
record.asctime = self.formatTime(record, self.datefmt)
return self._fmt.format(**record.__dict__)
DEFAULT_LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'propagate': False,
'formatters': {
'standard': {
'()' : standardFormatterFactory,
'fmt': '{asctime}-{levelname}: {message}',
'datefmt': '%Y-%m-%d %H:%M:%S',
'style': '{'
},
},
'handlers': {
'default': {
'level': 'DEBUG',
'formatter': 'standard',
'class': 'logging.StreamHandler',
'stream': 'ext://sys.stdout', # Default is stderr
},
},
'loggers': {
'__main__': { # if __name__ == '__main__'
'handlers': ['default'],
'level': 'DEBUG',
'formatter': 'standard'
},
}
}
logger = logging.getLogger(__name__)
logging.config.dictConfig(DEFAULT_LOGGING)
if __name__ == '__main__':
logger.info('Hello, {}', 'log')
sys.exit(0)
For Apache Airflow v1.10 running in Python2.7, with `pip install airflow[gcp_api] I am trying to setup logging for the Google Cloud. I have the following log_config py file:
GCS_LOG_FOLDER = 'gs://GCSbucket/'
LOG_LEVEL = conf.get('core', 'LOGGING_LEVEL').upper()
FAB_LOG_LEVEL = conf.get('core', 'FAB_LOGGING_LEVEL').upper()
LOG_FORMAT = conf.get('core', 'LOG_FORMAT')
BASE_LOG_FOLDER = conf.get('core', 'BASE_LOG_FOLDER')
PROCESSOR_LOG_FOLDER = conf.get('scheduler', 'CHILD_PROCESS_LOG_DIRECTORY')
FILENAME_TEMPLATE = conf.get('core', 'LOG_FILENAME_TEMPLATE')
PROCESSOR_FILENAME_TEMPLATE = conf.get('core', 'LOG_PROCESSOR_FILENAME_TEMPLATE')
# Storage bucket url for remote logging
# s3 buckets should start with "s3://"
# gcs buckets should start with "gs://"
# wasb buckets should start with "wasb"
# just to help Airflow select correct handler
REMOTE_BASE_LOG_FOLDER = conf.get('core', 'REMOTE_BASE_LOG_FOLDER')
LOGGING_CONFIG = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'airflow': {
'format': LOG_FORMAT,
},
},
'handlers': {
'console': {
'class': 'airflow.utils.log.logging_mixin.RedirectStdHandler',
'formatter': 'airflow',
'stream': 'sys.stdout'
},
'task': {
'class': 'airflow.utils.log.file_task_handler.FileTaskHandler',
'formatter': 'airflow',
'base_log_folder': os.path.expanduser(BASE_LOG_FOLDER),
'filename_template': FILENAME_TEMPLATE,
},
'processor': {
'class': 'airflow.utils.log.file_processor_handler.FileProcessorHandler',
'formatter': 'airflow',
'base_log_folder': os.path.expanduser(PROCESSOR_LOG_FOLDER),
'filename_template': PROCESSOR_FILENAME_TEMPLATE,
},
# Add a GCSTaskHandler to the 'handlers' block of the LOGGING_CONFIG variable
'gcs.task': {
'class': 'airflow.utils.log.gcs_task_handler.GCSTaskHandler',
'formatter': 'airflow',
'base_log_folder': os.path.expanduser(BASE_LOG_FOLDER),
'gcs_log_folder': GCS_LOG_FOLDER,
'filename_template': FILENAME_TEMPLATE,
},
},
'loggers': {
'airflow.processor': {
'handlers': ['processor'],
'level': LOG_LEVEL,
'propagate': False,
},
'airflow.task': {
'handlers': ['gcs.task'],
'level': LOG_LEVEL,
'propagate': False,
},
'airflow.task_runner': {
'handlers': ['gcs.task'],
'level': LOG_LEVEL,
'propagate': True,
},
'flask_appbuilder': {
'handler': ['console'],
'level': FAB_LOG_LEVEL,
'propagate': True,
}
},
'root': {
'handlers': ['console'],
'level': LOG_LEVEL,
}
}
REMOTE_HANDLERS = {
's3': {
'task': {
'class': 'airflow.utils.log.s3_task_handler.S3TaskHandler',
'formatter': 'airflow',
'base_log_folder': os.path.expanduser(BASE_LOG_FOLDER),
's3_log_folder': REMOTE_BASE_LOG_FOLDER,
'filename_template': FILENAME_TEMPLATE,
},
'processor': {
'class': 'airflow.utils.log.s3_task_handler.S3TaskHandler',
'formatter': 'airflow',
'base_log_folder': os.path.expanduser(PROCESSOR_LOG_FOLDER),
's3_log_folder': REMOTE_BASE_LOG_FOLDER,
'filename_template': PROCESSOR_FILENAME_TEMPLATE,
},
},
'gcs': {
'task': {
'class': 'airflow.utils.log.gcs_task_handler.GCSTaskHandler',
'formatter': 'airflow',
'base_log_folder': os.path.expanduser(BASE_LOG_FOLDER),
'gcs_log_folder': REMOTE_BASE_LOG_FOLDER,
'filename_template': FILENAME_TEMPLATE,
},
'processor': {
'class': 'airflow.utils.log.gcs_task_handler.GCSTaskHandler',
'formatter': 'airflow',
'base_log_folder': os.path.expanduser(PROCESSOR_LOG_FOLDER),
'gcs_log_folder': REMOTE_BASE_LOG_FOLDER,
'filename_template': PROCESSOR_FILENAME_TEMPLATE,
},
},
'wasb': {
'task': {
'class': 'airflow.utils.log.wasb_task_handler.WasbTaskHandler',
'formatter': 'airflow',
'base_log_folder': os.path.expanduser(BASE_LOG_FOLDER),
'wasb_log_folder': REMOTE_BASE_LOG_FOLDER,
'wasb_container': 'airflow-logs',
'filename_template': FILENAME_TEMPLATE,
'delete_local_copy': False,
},
'processor': {
'class': 'airflow.utils.log.wasb_task_handler.WasbTaskHandler',
'formatter': 'airflow',
'base_log_folder': os.path.expanduser(PROCESSOR_LOG_FOLDER),
'wasb_log_folder': REMOTE_BASE_LOG_FOLDER,
'wasb_container': 'airflow-logs',
'filename_template': PROCESSOR_FILENAME_TEMPLATE,
'delete_local_copy': False,
},
}
}
REMOTE_LOGGING = conf.get('core', 'remote_logging')
if REMOTE_LOGGING and REMOTE_BASE_LOG_FOLDER.startswith('s3://'):
LOGGING_CONFIG['handlers'].update(REMOTE_HANDLERS['s3'])
elif REMOTE_LOGGING and REMOTE_BASE_LOG_FOLDER.startswith('gs://'):
LOGGING_CONFIG['handlers'].update(REMOTE_HANDLERS['gcs'])
elif REMOTE_LOGGING and REMOTE_BASE_LOG_FOLDER.startswith('wasb'):
LOGGING_CONFIG['handlers'].update(REMOTE_HANDLERS['wasb'])
My airflow.cfg settings are:
[core]
remote_logging = True
remote_base_log_folder = gs:/GCSbucket/logs
remote_log_conn_id = google_cloud_default
The error I get is the following:
Traceback (most recent call last):
File "/usr/local/lib/python2.7/atexit.py", line 24, in _run_exitfuncs
func(*targs, **kargs)
File "/usr/local/lib/python2.7/logging/__init__.py", line 1676, in shutdown
h.close()
File "/usr/local/lib/python2.7/site-packages/airflow/utils/log/gcs_task_handler.py", line 73, in close
if self.closed:
AttributeError: 'GCSTaskHandler' object has no attribute 'closed'
Does anybody know what might have gone wrong?
The tutorial that is being followed is: https://airflow.readthedocs.io/en/1.10.0/howto/write-logs.html
Update: Did some more research in the source code, here I see that the close statement returns nothing, and this is why my application crashes.
https://github.com/apache/incubator-airflow/blob/v1-10-stable/airflow/utils/log/gcs_task_handler.py
Does somebody know why nothing is returend in
def close(self):
if self.closed:
return
The instructions might be outdated. Please try with the instructions from the following link:
https://airflow.readthedocs.io/en/latest/howto/write-logs.html#writing-logs-to-google-cloud-storage
Follow the steps below to enable Google Cloud Storage logging.
To enable this feature, airflow.cfg must be configured as in this example:
[core]
# Airflow can store logs remotely in AWS S3, Google Cloud Storage or Elastic Search.
# Users must supply an Airflow connection id that provides access to the storage
# location. If remote_logging is set to true, see UPDATING.md for additional
# configuration requirements.
remote_logging = True
remote_base_log_folder = gs://my-bucket/path/to/logs
remote_log_conn_id = MyGCSConn
Install the gcp_api package first, like so: pip install apache-airflow[gcp_api].
Make sure a Google Cloud Platform connection hook has been defined in Airflow. The hook should have read and write access to the Google Cloud Storage bucket defined above in remote_base_log_folder.
Restart the Airflow webserver and scheduler, and trigger (or wait for) a new task execution.
Verify that logs are showing up for newly executed tasks in the bucket you’ve defined.
Verify that the Google Cloud Storage viewer is working in the UI. Pull up a newly executed task, and verify that you see something like:
-
*** Reading remote log from gs://<bucket where logs should be persisted>/example_bash_operator/run_this_last/2017-10-03T00:00:00/16.log.
[2017-10-03 21:57:50,056] {cli.py:377} INFO - Running on host chrisr-00532
[2017-10-03 21:57:50,093] {base_task_runner.py:115} INFO - Running: ['bash', '-c', u'airflow run example_bash_operator run_this_last 2017-10-03T00:00:00 --job_id 47 --raw -sd DAGS_FOLDER/example_dags/example_bash_operator.py']
[2017-10-03 21:57:51,264] {base_task_runner.py:98} INFO - Subtask: [2017-10-03 21:57:51,263] {__init__.py:45} INFO - Using executor SequentialExecutor
[2017-10-03 21:57:51,306] {base_task_runner.py:98} INFO - Subtask: [2017-10-03 21:57:51,306] {models.py:186} INFO - Filling up the DagBag from /airflow/dags/example_dags/example_bash_operator.py
To resolve this question I added the following to the core of airflow.cfg
[core]
log_filename_template = {{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log
log_processor_filename_template = {{ filename }}.log
# Log format
# we need to escape the curly braces by adding an additional curly brace
log_format = [%%(asctime)s] {%%(filename)s:%%(lineno)d} %%(levelname)s - %%(message)s
simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s
# Logging class
# Specify the class that will specify the logging configuration
# This class has to be on the python classpath
# logging_config_class = my.path.default_local_settings.LOGGING_CONFIG
logging_config_class =log_config.LOGGING_CONFIG
task_log_reader = gcs.task
At the log_config.LOGGING_CONFIG I added the following:
# Add a GCSTaskHandler to the 'handlers' block of the LOGGING_CONFIG variable
'gcs.task': {
'class': 'airflow.utils.log.gcs_task_handler.GCSTaskHandler',
'formatter': 'airflow',
'base_log_folder': os.path.expanduser(BASE_LOG_FOLDER),
'gcs_log_folder': GCS_LOG_FOLDER,
'filename_template': FILENAME_TEMPLATE,
}
The tutorial to be followed is:
https://airflow.readthedocs.io/en/1.10.0/howto/write-logs.html
Environment:
OS: redhat
python version: 3.6
django: 2.1
django channels: 2.1.3
The following is my logging.conf file (same level as settings.py):
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'standard': {
'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s',
'datefmt': '%Y-%m-%d %H:%M:%S',
},
},
'handlers': {
'file': {
'level': 'DEBUG',
'class': 'logging.FileHandler',
'filename': 'debug.log',
'formatter': 'standard'
},
},
'loggers': {
'django': {
'handlers': ['file'],
'level': 'DEBUG',
'propagate': True,
},
},
}
django_logger = logging.getLogger('django')
and the function to log as follows:
def log_it(*args):
try:
django_logger.info(str(args).encode('utf-8'))
return True
except:
django_logger.info('#### Exception in LOGGING!!!!!!')
return False
I am using this to log inside django consumers as follows:
class EchoConsumer(SyncConsumer):
def websocket_connect(self, event):
try:
log_it('inside EchoConsumer connect()', event)
self.send({
"type": "websocket.accept",
})
return True
except:
error = traceback.format_exc()
write_error_log(error)
return False
def websocket_receive(self, event):
try:
log_it('inside EchoConsumer receive()', event)
self.send({
"type": "websocket.send",
"text": event["text"],
})
return True
except:
error = traceback.format_exc()
write_error_log(error)
return False
The log_it() used inside the consumers aren't being invoked.
(Assumption: Integrating channels with django must have overriden the django logging settings)
I checked if there was any problem in reaching the consumers in the daphne access logs.
I also checked my nginx conf, but the connection requests were recorded as follows:
127.0.0.1:45100 - - [08/Oct/2018:19:00:58] "WSCONNECTING /ws/event/" - -
127.0.0.1:45100 - - [08/Oct/2018:19:00:58] "WSCONNECT /ws/event/" - -
Note: I have linked /ws/event to EchoConsumer in routing.py
My expectation is:
127.0.0.1:45100 - - [08/Oct/2018:19:00:58] "WSCONNECTING /ws/event/" - -
127.0.0.1:45100 - - [08/Oct/2018:19:00:58] "WSCONNECT /ws/event/" - -
inside EchoConsumer connect() ....
Any idea why the log statements are not being displayed?
P.S related issue on github
I have a logger that just adds log_line if none is provided when calling logger.debug
class UrlLogger(logging.Logger):
def _log(self, level, msg, args, exc_info=None, extra=None):
if extra is None:
extra = {'log_line':' '}
super(UrlLogger, self)._log(level, msg, args, exc_info, extra)
I have about 20 different modules that I need to go into and add
logging.setLoggerClass(UrlLogger)
How can I set this as default in the settings?
My settings currently look like this:
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'verbose': {
'format': '%(levelname)s %(name)s %(asctime)s %(module)s %(process)d %(thread)d %(pathname)s#%(lineno)s: %(message)s'
},
'simple': {
'format': '[%(levelname)8s] [%(asctime)s] %(module)10s/%(filename)s:%(log_line)s - %(message)s',
'datefmt': '%d-%m-%Y %H:%M:%S'
},
},
'handlers': {
'console': {
'class': 'logging.StreamHandler',
'level': LOG_LEVEL,
'formatter': 'simple'
},
},
'loggers': {
'': {
'level': LOG_LEVEL,
'handlers': ['console'],
'class': ['UrlLogger'],
},
As you can see, I tried setting it here, but I don't think that's working because I am still getting errors.
Message: 'Not Found: %s'
Arguments: ('/tool_job_list/1/1fa119372de6/job/check/jexecution/report',)
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.6/logging/__init__.py", line 992, in emit
msg = self.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 838, in format
return fmt.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 578, in format
s = self.formatMessage(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 547, in formatMessage
return self._style.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 391, in format
return self._fmt % record.__dict__
KeyError: 'log_line'
This error occurs when the front end makes failed API calls (wrong uri) to the django backend. I'm thinking that the failed api call is logging the error without using my UrlLogger (otherwise there would not be a key error unless there is an error in my Logger Class).
What am I doing wrong here?
I am trying to set up logging where I can log in both stdout and on to a file. This i have accomplished using the following code:
logging.basicConfig(
level=logging.DEBUG, format='%(asctime)-15s %(levelname)-8s %(message)s',
datefmt='%a, %d %b %Y %H:%M:%S', handlers=[logging.FileHandler(path), logging.StreamHandler()])
The output of this something like this:
2018-05-02 18:43:33,295 DEBUG Starting new HTTPS connection (1): google.com
2018-05-02 18:43:33,385 DEBUG https://google.com:443 "GET / HTTP/1.1" 301 220
2018-05-02 18:43:33,389 DEBUG Starting new HTTPS connection (1): www.google.com
2018-05-02 18:43:33,490 DEBUG https://www.google.com:443 "GET / HTTP/1.1" 200 None
What I am trying to accomplish is logging this output to a file not as it is printing to stdout, but as a dictionary or JSON object similar to something like this (while keeping the stdout as it is at the moment):
[{'time': '2018-05-02 18:43:33,295', 'level': 'DEBUG', 'message': 'Starting new HTTPS connection (1): google.com'}, {...}, {...}]
Is this doable? I understand that I can post process this log file after my process is finished, but I am looking for a more elegant solution because certain things i am logging are quite big objects themselves.
I too dealt with this and I personally believe that an external library might be an overkill for something like this.
I studied a bit the code behind logging.Formatter and came up with a subclass which in my case does the trick (my goal was to have a JSON file that Filebeat can read to further log into ElasticSearch).
Class:
import logging
import json
class JsonFormatter(logging.Formatter):
"""
Formatter that outputs JSON strings after parsing the LogRecord.
#param dict fmt_dict: Key: logging format attribute pairs. Defaults to {"message": "message"}.
#param str time_format: time.strftime() format string. Default: "%Y-%m-%dT%H:%M:%S"
#param str msec_format: Microsecond formatting. Appended at the end. Default: "%s.%03dZ"
"""
def __init__(self, fmt_dict: dict = None, time_format: str = "%Y-%m-%dT%H:%M:%S", msec_format: str = "%s.%03dZ"):
self.fmt_dict = fmt_dict if fmt_dict is not None else {"message": "message"}
self.default_time_format = time_format
self.default_msec_format = msec_format
self.datefmt = None
def usesTime(self) -> bool:
"""
Overwritten to look for the attribute in the format dict values instead of the fmt string.
"""
return "asctime" in self.fmt_dict.values()
def formatMessage(self, record) -> dict:
"""
Overwritten to return a dictionary of the relevant LogRecord attributes instead of a string.
KeyError is raised if an unknown attribute is provided in the fmt_dict.
"""
return {fmt_key: record.__dict__[fmt_val] for fmt_key, fmt_val in self.fmt_dict.items()}
def format(self, record) -> str:
"""
Mostly the same as the parent's class method, the difference being that a dict is manipulated and dumped as JSON
instead of a string.
"""
record.message = record.getMessage()
if self.usesTime():
record.asctime = self.formatTime(record, self.datefmt)
message_dict = self.formatMessage(record)
if record.exc_info:
# Cache the traceback text to avoid converting it multiple times
# (it's constant anyway)
if not record.exc_text:
record.exc_text = self.formatException(record.exc_info)
if record.exc_text:
message_dict["exc_info"] = record.exc_text
if record.stack_info:
message_dict["stack_info"] = self.formatStack(record.stack_info)
return json.dumps(message_dict, default=str)
Usage:
The formatter must simply be passed to the logging handler.
json_handler = FileHandler("foo.json")
json_formatter = JsonFormatter({"level": "levelname",
"message": "message",
"loggerName": "name",
"processName": "processName",
"processID": "process",
"threadName": "threadName",
"threadID": "thread",
"timestamp": "asctime"})
json_handler.setFormatter(json_formatter)
Explanation :
While the logging.Formatter takes a string which it interpolates to output the formatted log record, the JsonFormatter takes a dictionary where the key will be the key of the logged value in the JSON string and the value is a string corresponding to an attribute of the LogRecord that can be logged.
(List available in the docs here)
Main "problem" would be parsing dates and timestamps, and the default formatter implementation has these class attributes, default_time_format and default_msec_format.
default_msec_format gets passed to time.strftime() and default_msec_format is interpolated to append miliseconds as time.strftime() doesn't provide formatting options for those.
The principle is that those are now instance attributes which can be provided in the form of time_format and msec_format to customize how the parent's class (unchanged, as it's not overwritten) formatTime() method behaves.
You could technically override it if you want to customize time formatting, but I personally found that using something else would either be redundant or limit the actual formatting options. But feel free to adjust as to your needs.
Output:
An example JSON record logged by the formatting options above, with the default time formatting options set in the class, would be:
{"level": "INFO", "message": "Starting service...", "loggerName": "root", "processName": "MainProcess", "processID": 25103, "threadName": "MainThread", "threadID": 4721200640, "timestamp": "2021-12-04T08:25:07.610Z"}
So based on #abarnert, i found this Link which provided a good path to making this concept work for the most part. The code as it stands is:
logger=logging.getLogger()
logger.setLevel(logging.DEBUG)
file_handler=logging.FileHandler('foo.log')
stream_handler=logging.StreamHandler()
stream_formatter=logging.Formatter(
'%(asctime)-15s %(levelname)-8s %(message)s')
file_formatter=logging.Formatter(
"{'time':'%(asctime)s', 'name': '%(name)s', \
'level': '%(levelname)s', 'message': '%(message)s'}"
)
file_handler.setFormatter(file_formatter)
stream_handler.setFormatter(stream_formatter)
logger.addHandler(file_handler)
logger.addHandler(stream_handler)
Although it does not fully meet the requirement, it doesnt require any pre processing, and allows me to create two log handlers.
Afterwards, i can use something like:
with open('foo.log') as f:
logs = f.read().splitlines()
for l in logs:
for key, value in eval(l):
do something ...
to pull dict objects instead of fighting with improperly formatted JSON to accomplish what i had set out to accomplish.
Still am hoping for a more elegant solution.
I wanted to have JSON output, so that I could handle it nicer in Promtail and Loki. I just updated the formatter in my logging.json, which I use as a dictConfig
"formatters": {
"normalFormatter": {
"format": "{\"time\": \"%(asctime)s\", \"name\": \"[%(name)s]\", \"levelname\": \"%(levelname)s\", \"message\": \"%(message)s\"}"
}
}
Load the config and get the root logger like:
import json
import logging
# setup logger
with open("logging.json") as f:
config_dict = json.load(f)
logging.config.dictConfig(config_dict)
# get root logger
logger = logging.getLogger(__name__)
If you want to use dictConfig, please make sure you gave all necessary fields.
https://docs.python.org/3/library/logging.config.html
You may want to define a formatter, a handler (which is using the formatter) and the logger (which is using the handler)
e.g. logging.json
{
"version": 1,
"disable_existing_loggers": false,
"formatters": {
"normalFormatter": {
"format": "{\"time\": \"%(asctime)s\", \"name\": \"[%(name)s]\", \"levelname\": \"%(levelname)s\", \"message\": \"%(message)s\"}"
}
},
"handlers": {
"demohandler": {
"level": "INFO",
"formatter": "normalFormatter",
"class": "logging.handlers.TimedRotatingFileHandler",
"filename": "./files/logs/YourLogFile.log",
"when": "d",
"interval": 30,
"backupCount": 4,
"utc": true
}
},
"loggers": {
"root": {
"handlers": ["demohandler"],
"level": "INFO"
},
"someModule": {
"handlers": ["demohandler"],
"level": "INFO",
"propagate": 0
}
}
}
With this code you can add the full traceback, timestamp and level to a json file of choice.
import json
import traceback
from datetime import datetime
def addLogging(logDict:dict):
loggingsFile = 'loggings.json'
with open(loggingsFile) as f:
data = json.load(f)
data.append(logDict)
with open(loggingsFile, 'w') as f:
json.dump(data, f)
def currentTimeUTC():
return datetime.now().strftime('%d/%m/%Y %H:%M:%S')
try:
print(5 / 0)
except ZeroDivisionError:
fullTraceback = str(traceback.format_exc())
addLogging({'timestamp': currentTimeUTC(), 'level': 'error', 'traceback': fullTraceback})
Output:
[
{
"timestamp": "09/06/2020 17:38:00",
"level": "error",
"traceback": "Traceback (most recent call last):\n File \"d:testFullTraceback.py\", line 19, in <module>\n print(5/0)\nZeroDivisionError: division by zero\n"
}
]
I could achieve this result using this custom formatter:
import json
import logging
class CustomJsonFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
super(CustomJsonFormatter, self).format(record)
output = {k: str(v) for k, v in record.__dict__.items()}
return json.dumps(output)
cf = CustomJsonFormatter()
sh = logging.StreamHandler()
sh.setFormatter(cf)
logger = logging.getLogger("my.module")
logger.addHandler(sh)
# simple json output
logger.warning("This is a great %s!", "log")
# enrich json output
logger.warning("This is an even greater %s!", "log", extra={'foo': 'bar'})
Output:
{"name": "my.module", "msg": "This is a great %s!", "args": "('log',)", "levelname": "WARNING", "levelno": "30", "pathname": "/Users/olivier/test.py", "filename": "test.py", "module": "test", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "20", "funcName": "<module>", "created": "1661868378.5048351", "msecs": "504.8351287841797", "relativeCreated": "1.3060569763183594", "thread": "4640826880", "threadName": "MainThread", "processName": "MainProcess", "process": "81360", "message": "This is a great log!"}
{"name": "my.module", "msg": "This is an even greater %s!", "args": "('log',)", "levelname": "WARNING", "levelno": "30", "pathname": "/Users/olivier/test.py", "filename": "test.py", "module": "test", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "22", "funcName": "<module>", "created": "1661868378.504962", "msecs": "504.9619674682617", "relativeCreated": "1.4328956604003906", "thread": "4640826880", "threadName": "MainThread", "processName": "MainProcess", "process": "81360", "foo": "bar", "message": "This is an even greater log!"}
If you don't mind pip installing a module to help, there is the json_log_formatter module. The json output does have more attributes than requested. The repo mentions customizing the attributes outputted, but I have not yet integrated that into a working example just yet.
json_log_formatter
import logging
import json_log_formatter
# Set Basic Logging
self.loggers = logging.getLogger(__name__)
self.loggers.setLevel(logging.DEBUG)
self.formatter = logging.Formatter(fmt='%(asctime)-15s %(levelname)-8s %(message)s', datefmt = '%a, %d %b %Y %H:%M:%S')
# Config for JSON File Handler
self.logFileHandler = logging.FileHandler(SOME-PATH, mode='a')
self.fileFormatter = json_log_formatter.VerboseJSONFormatter()
self.logFileHandler.setFormatter(self.fileFormatter)
self.logFileHandler.setLevel(logging.INFO)
self.loggers.addHandler(self.logFileHandler)
# Config for Stream Handler
self.logStreamHandler = logging.StreamHandler()
self.logStreamHandler.setFormatter(self.formatter)
self.logStreamHandler.setLevel(logging.INFO)
self.loggers.addHandler(self.logStreamHandler)
I wanted to save full LogRecord object, so I can later inspect my log with maximum of integration with module. So I inspected object like that:
class Handler_json(Handler):
def emit(self, record: LogRecord) -> None:
json_data = {}
for attr in filter(lambda attr: not attr.endswith("__"), dir(record)):
json_data[attr] = record.__getattribute__(attr)
del json_data["getMessage"]
print(json_data)
this is subclass of Handler, emit is rewritten method that is been called with every LogRecord. Dir returns all attributes and methods of the object. I am excluding special methods, and also deleting getMessage method, whitch is not needed for json object representation.
This can be nicely integrated in logging like that:
logger = getLogger(__name__)
logger.setLevel(DEBUG)
handle_json = Handler_json()
logger.addHandler(handle_json)
logger.info("my info")
result looks like this:
{
'args': (),
'created': 1639925351.0648422,
'exc_info': None,
'exc_text': None,
'filename': 'my_logging.py',
'funcName': 'restore_log_from_disk',
'levelname': 'INFO',
'levelno': 20,
'lineno': 142,
'module': 'my_logging',
'msecs': 64.84222412109375,
'msg': 'my info',
'name': '__main__',
'pathname':
'/home/jindrich/PycharmProjects/my_logging.py',
'process': 146331,
'processName': 'MainProcess',
'relativeCreated': 1.6417503356933594,
'stack_info': None,
'thread': 140347192436544,
'threadName': 'MainThread'
}
then you can load data from disk and recreat objects after some digging in doc.
I was able to create it using python-json-logger lib, it was simple and really easily to use.
Django
from pythonjsonlogger import jsonlogger
##This is to add custom keys
class CustomJsonFormatter(jsonlogger.JsonFormatter):
def add_fields(self, log_record, record, message_dict):
super(CustomJsonFormatter, self).add_fields(log_record, record, message_dict)
log_record['level'] = record.levelname
# Logging
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'json': {
'()': CustomJsonFormatter, # if you want to use custom logs class defined above
# '()': jsonlogger.JsonFormatter, # without custom logs
'format': '%(level)s %(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s %(threadName)s %(special)s %(run)s\
%(name)s %(created)s %(processName)s %(relativeCreated)d %(funcName)s %(levelno)d %(msecs)d %(pathname)s %(lineno)d %(filename)s'
},
},
'handlers': {
'null': {
'class': 'logging.NullHandler',
},
'console': {
'level': 'DEBUG',
'class': 'logging.StreamHandler',
'formatter': 'json'
},
},
.....
}
Flask
from logging.config import dictConfig
from pythonjsonlogger import jsonlogger
import os
# This will set global root logging config across all the modules using in the app
##This is to add custom keys
class CustomJsonFormatter(jsonlogger.JsonFormatter):
def add_fields(self, log_record, record, message_dict):
super(CustomJsonFormatter, self).add_fields(log_record, record, message_dict)
log_record['level'] = record.levelname
def setup():
LOG_FILE = '/tmp/app/app.json'
if not os.path.exists(os.path.dirname(LOG_FILE)): # if LOG_FILE dir doesn't exist, creates it.
os.makedirs(os.path.dirname(LOG_FILE))
dictConfig({
'version': 1,
'formatters': {
'default': {
'format': '%(asctime)s - %(module)s - %(levelname)s - %(message)s',
},
'json': {
'()': CustomJsonFormatter, # if you want to use custom logs class defined above
# '()': jsonlogger.JsonFormatter, # without custom logs
'format': '%(level)s %(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s %(threadName)s %(special)s %(run)s\
%(name)s %(created)s %(processName)s %(relativeCreated)d %(funcName)s %(levelno)d %(msecs)d %(pathname)s %(lineno)d %(filename)s'
},
},
'handlers': {'file': {
'class': 'logging.handlers.RotatingFileHandler',
'filename': LOG_FILE,
'maxBytes': 10485760,
'backupCount': 5,
'formatter': 'json'
},
'console':{
'class':'logging.StreamHandler',
'formatter': 'json'
}},
'root': {
'level': 'INFO',
'handlers': ['file', 'console']
}
})
Hope this helps easy way setup.