I am logging my database queries in Django along with the pathname and linenumber.
Right now i am getting these logs:
07/Dec/2018 14:25:00 DEBUG django.db.backends utils **/Users/XXXXX/.idea/lib/python2.7/site-packages/django/db/backends/utils.py:89**
(0.340) SELECT "metadata"."metaname", "metadata"."description", "metadata"."attributes" FROM "metadata" WHERE "metadata"."metaname" = 'date_type'; args=('date_type',)
For all queries, I am getting the same path and line number. Is there any way I can capture the line number from my main application instead of the one from utils.
Current logging Implementation:
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'handlers': {
'console': {
'level': 'DEBUG',
'class': 'logging.StreamHandler',
'formatter': 'color'
},
},
'loggers': {
'django.db.backends': {
'handlers': ['console'],
'level': 'DEBUG',
'propogate': True,
}
}
}
Using python 2.7 and django 1.9
The slightly optimized version from User #will-keeling
Logging configuration for Django for output line numbers for each db request.
Note: If you want to use it for tests you need to set DEBUG=True for tests How do you set DEBUG to True when running a Django test?
import logging
import traceback
from django.conf import settings
class StackInfoHandler(logging.StreamHandler):
trim = 5
def emit(self, record):
super(StackInfoHandler, self).emit(record)
trace = traceback.format_stack()
stack1 = [str(row) for row in trace]
stack2 = [s for s in stack1 if settings.BASE_DIR in s and 'format_stack' not in s]
stack3 = [s for s in stack2 if 'test' not in s]
if not stack3:
stack3 = stack2 # include test call
if stack3:
stack4 = ''.join(stack3[-self.trim:]) # take only last records
stack5 = f"Stack {self.terminator} {''.join(stack4)}"
self.stream.write(stack5)
self.stream.write(self.terminator)
self.flush()
Logging Config (partitial)
LOGGING = {
'handlers': {
'db-console': {
'level': 'DEBUG',
'class': 'settings.local.StackInfoHandler', # Reference the custom handler
'formatter': 'simple',
},
'loggers': {
'django.db.backends': {
'handlers': ['db-console'],
'level': 'DEBUG',
'propagate': False
},
}
}
}
This will show you only stack trace from your Django codebase like below
[2020-05-25 17:49:17,977]: (0.000) INSERT INTO `contacts_contactscount` (`user_id`, `date`, `amount`) VALUES (338, '2020-05-25 17:49:17', 7); args=[338, '2020-05-25 17:49:17', 7]
Stack
File "<project-root>/api/views/contacts.py", line 164, in create
Contact.objects.filter(pk__in=to_delete).delete()
File "<project-root>/<folder>/contacts/models.py", line 54, in delete
create_deletion_log.delay(obj, deleted_timestamp)
File "<project-root>/<folder>/contacts/tasks.py", line 31, in create_deletion_log
contact.save()
File "<project-root>/<folder>/contacts/models.py", line 118, in save
Contact.objects.contacts_added_hook(self.user)
File "<project-root>/<folder>/contacts/models.py", line 67, in contacts_added_hook
current_total = user.profile.contacts_total
File "<project-root>/<folder>/profile/models.py", line 631, in contacts_total
ContactsCount.objects.create(user=self.user, amount=count)
I'm guessing that you're trying to determine which lines in your application are responsible for running which queries.
One way to achieve this would be to create a custom handler that prints out the current stack at the point where Django logs the query. That would allow you to see which line in your application is executing.
You could create a custom handler such as:
import logging
import traceback
class StackInfoHandler(logging.StreamHandler):
trim = 5
def emit(self, record):
super(StackInfoHandler, self).emit(record)
stack = ''.join(
str(row) for row in traceback.format_stack()[:-self.trim]
)
self.stream.write(stack)
And then in your logging config, you could just switch the handler class to use the StackInfoHandler:
'handlers': {
'console': {
'level': 'DEBUG',
'class': 'my.package.StackInfoHandler', # Reference the custom handler
'formatter': 'color'
},
},
Note that the StackInfoHandler trims 5 lines off the stack so that it doesn't show you stack frames from the logging framework itself. You might need to tweak this number (5 works for me locally).
Environment:
OS: redhat
python version: 3.6
django: 2.1
django channels: 2.1.3
The following is my logging.conf file (same level as settings.py):
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'standard': {
'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s',
'datefmt': '%Y-%m-%d %H:%M:%S',
},
},
'handlers': {
'file': {
'level': 'DEBUG',
'class': 'logging.FileHandler',
'filename': 'debug.log',
'formatter': 'standard'
},
},
'loggers': {
'django': {
'handlers': ['file'],
'level': 'DEBUG',
'propagate': True,
},
},
}
django_logger = logging.getLogger('django')
and the function to log as follows:
def log_it(*args):
try:
django_logger.info(str(args).encode('utf-8'))
return True
except:
django_logger.info('#### Exception in LOGGING!!!!!!')
return False
I am using this to log inside django consumers as follows:
class EchoConsumer(SyncConsumer):
def websocket_connect(self, event):
try:
log_it('inside EchoConsumer connect()', event)
self.send({
"type": "websocket.accept",
})
return True
except:
error = traceback.format_exc()
write_error_log(error)
return False
def websocket_receive(self, event):
try:
log_it('inside EchoConsumer receive()', event)
self.send({
"type": "websocket.send",
"text": event["text"],
})
return True
except:
error = traceback.format_exc()
write_error_log(error)
return False
The log_it() used inside the consumers aren't being invoked.
(Assumption: Integrating channels with django must have overriden the django logging settings)
I checked if there was any problem in reaching the consumers in the daphne access logs.
I also checked my nginx conf, but the connection requests were recorded as follows:
127.0.0.1:45100 - - [08/Oct/2018:19:00:58] "WSCONNECTING /ws/event/" - -
127.0.0.1:45100 - - [08/Oct/2018:19:00:58] "WSCONNECT /ws/event/" - -
Note: I have linked /ws/event to EchoConsumer in routing.py
My expectation is:
127.0.0.1:45100 - - [08/Oct/2018:19:00:58] "WSCONNECTING /ws/event/" - -
127.0.0.1:45100 - - [08/Oct/2018:19:00:58] "WSCONNECT /ws/event/" - -
inside EchoConsumer connect() ....
Any idea why the log statements are not being displayed?
P.S related issue on github
Related to a previous question I asked, Airbrake logger in Django still sending notifications even though its level is set to 'CRITICAL'?, I'm working on a Django project with multiple versions of settings.py: settings/base.py, settings/staging.py, etc.
There is also a kind of settings 'mixin', settings/staging_development.py, which contains the following LOGGING configuration:
# Auxiliary variable used in LOGGING
_AIRBRAKE_LOGGER = {
'handlers': ['airbrake'],
'level': 'ERROR',
'propagate': True,
}
# Airbrake logging integration (cf. https://github.com/airbrake/pybrake#django-integration)
# In our case, 'app' is replaced by three apps, 'lucy_web', 'api', and 'activation'.
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'handlers': {
'airbrake': {
'level': 'ERROR',
'class': 'pybrake.LoggingHandler',
},
},
'loggers': {
'lucy_web': _AIRBRAKE_LOGGER,
'api': _AIRBRAKE_LOGGER,
'activation': _AIRBRAKE_LOGGER,
},
}
This is imported in settings/staging.py in which LOGGING is further .update()d as follows:
from lucy.settings.staging_production import *
# LOGGING = {
# 'version': 1,
# 'disable_existing_loggers': False,
# 'handlers': {
# 'console': {
# 'class': 'logging.StreamHandler',
# },
# },
# 'loggers': {
# 'django': {
# 'handlers': ['console'],
# 'level': os.getenv('LOG_LEVEL', 'INFO'),
# },
# },
# }
LOGGING['handlers'].update(console={
'class': 'logging.StreamHandler'
})
LOGGING['loggers'].update(django={
'handlers': ['console'],
'level': os.getenv('LOG_LEVEL', 'INFO'),
})
What I'd like to achieve is to log to Airbrake as well as to the console. Now, if I simply comment in the commented-out code and re-define the LOGGING configuration, I notice that error messages are successfully getting logged to the console. If I use the code as-is, however, they do not, even though I do get Airbrake notifications.
If I poke around in the shell, everything looks OK: for example, the 'lucy_web' and 'django' loggers both have handlers attached and propagate set to True:
(venv) Kurts-MacBook-Pro-2:lucy-web kurtpeek$ ENV_ROLE=staging_on_localhost python manage.py shell
Python 3.6.4 (v3.6.4:d48ecebad5, Dec 18 2017, 21:07:28)
Type 'copyright', 'credits' or 'license' for more information
IPython 6.3.1 -- An enhanced Interactive Python. Type '?' for help.
In [1]: from django.conf import settings
In [2]: settings.LOGGING
Out[2]:
{'version': 1,
'disable_existing_loggers': False,
'handlers': {'airbrake': {'level': 'ERROR',
'class': 'pybrake.LoggingHandler'},
'console': {'class': 'logging.StreamHandler'}},
'loggers': {'lucy_web': {'handlers': ['airbrake'],
'level': 'ERROR',
'propagate': True},
'api': {'handlers': ['airbrake'], 'level': 'ERROR', 'propagate': True},
'activation': {'handlers': ['airbrake'],
'level': 'ERROR',
'propagate': True},
'django': {'handlers': ['console'], 'level': 'ERROR'}}}
In [3]: import logging
In [4]: logger = logging.getLogger('lucy_web')
In [6]: logger.propagate
Out[6]: True
In [7]: logger.hasHandlers()
Out[7]: True
In [8]: logger.handlers
Out[8]: [<LoggingHandler (ERROR)>]
In [9]: django_logger = logging.getLogger('django')
In [10]: django_logger.handlers
Out[10]: [<StreamHandler <stderr> (NOTSET)>]
In [11]: django_logger.getEffectiveLevel()
Out[11]: 40
In [12]: logging.ERROR
Out[12]: 40
In [13]: django_logger.propagate
Out[13]: True
In short, I can get one type of logging to work or the other, but not both at the same time, even though log propagation is enabled.
Any idea what the issue might be?
I resolved the problem by adding the 'console' handler to the apps ('lucy_web', 'api', and 'activation') themselves, rather than to the catch-all 'django' logger. Actually, I think that at the root of the problem, the errors I was throwing were not in 'django' logger's hierarchy.
So firstly, in settings/base.py I use deepcopy:
from copy import deepcopy
# Auxiliary variable used in LOGGING
_AIRBRAKE_LOGGER = {
'handlers': ['airbrake'],
'level': 'ERROR',
'propagate': True,
}
# Airbrake logging integration (cf. https://github.com/airbrake/pybrake#django-integration)
# In our case, 'app' is replaced by three apps, 'lucy_web', 'api', and 'activation'.
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'handlers': {
'airbrake': {
'level': 'ERROR',
'class': 'pybrake.LoggingHandler',
},
},
'loggers': {
# The deepcopy allows us to append to each app's 'handlers' list without affecting the others
'lucy_web': deepcopy(_AIRBRAKE_LOGGER),
'api': deepcopy(_AIRBRAKE_LOGGER),
'activation': deepcopy(_AIRBRAKE_LOGGER),
},
}
And secondly, in settings/staging.py I updated LOGGING like so:
LOGGING['handlers'].update(console={
'class': 'logging.StreamHandler'
})
for app in ('lucy_web', 'api', 'activation'):
LOGGING['loggers'][app]['handlers'].append('console')
Now error gets logged both to Airbrake and to the console.
I am trying to set up logging where I can log in both stdout and on to a file. This i have accomplished using the following code:
logging.basicConfig(
level=logging.DEBUG, format='%(asctime)-15s %(levelname)-8s %(message)s',
datefmt='%a, %d %b %Y %H:%M:%S', handlers=[logging.FileHandler(path), logging.StreamHandler()])
The output of this something like this:
2018-05-02 18:43:33,295 DEBUG Starting new HTTPS connection (1): google.com
2018-05-02 18:43:33,385 DEBUG https://google.com:443 "GET / HTTP/1.1" 301 220
2018-05-02 18:43:33,389 DEBUG Starting new HTTPS connection (1): www.google.com
2018-05-02 18:43:33,490 DEBUG https://www.google.com:443 "GET / HTTP/1.1" 200 None
What I am trying to accomplish is logging this output to a file not as it is printing to stdout, but as a dictionary or JSON object similar to something like this (while keeping the stdout as it is at the moment):
[{'time': '2018-05-02 18:43:33,295', 'level': 'DEBUG', 'message': 'Starting new HTTPS connection (1): google.com'}, {...}, {...}]
Is this doable? I understand that I can post process this log file after my process is finished, but I am looking for a more elegant solution because certain things i am logging are quite big objects themselves.
I too dealt with this and I personally believe that an external library might be an overkill for something like this.
I studied a bit the code behind logging.Formatter and came up with a subclass which in my case does the trick (my goal was to have a JSON file that Filebeat can read to further log into ElasticSearch).
Class:
import logging
import json
class JsonFormatter(logging.Formatter):
"""
Formatter that outputs JSON strings after parsing the LogRecord.
#param dict fmt_dict: Key: logging format attribute pairs. Defaults to {"message": "message"}.
#param str time_format: time.strftime() format string. Default: "%Y-%m-%dT%H:%M:%S"
#param str msec_format: Microsecond formatting. Appended at the end. Default: "%s.%03dZ"
"""
def __init__(self, fmt_dict: dict = None, time_format: str = "%Y-%m-%dT%H:%M:%S", msec_format: str = "%s.%03dZ"):
self.fmt_dict = fmt_dict if fmt_dict is not None else {"message": "message"}
self.default_time_format = time_format
self.default_msec_format = msec_format
self.datefmt = None
def usesTime(self) -> bool:
"""
Overwritten to look for the attribute in the format dict values instead of the fmt string.
"""
return "asctime" in self.fmt_dict.values()
def formatMessage(self, record) -> dict:
"""
Overwritten to return a dictionary of the relevant LogRecord attributes instead of a string.
KeyError is raised if an unknown attribute is provided in the fmt_dict.
"""
return {fmt_key: record.__dict__[fmt_val] for fmt_key, fmt_val in self.fmt_dict.items()}
def format(self, record) -> str:
"""
Mostly the same as the parent's class method, the difference being that a dict is manipulated and dumped as JSON
instead of a string.
"""
record.message = record.getMessage()
if self.usesTime():
record.asctime = self.formatTime(record, self.datefmt)
message_dict = self.formatMessage(record)
if record.exc_info:
# Cache the traceback text to avoid converting it multiple times
# (it's constant anyway)
if not record.exc_text:
record.exc_text = self.formatException(record.exc_info)
if record.exc_text:
message_dict["exc_info"] = record.exc_text
if record.stack_info:
message_dict["stack_info"] = self.formatStack(record.stack_info)
return json.dumps(message_dict, default=str)
Usage:
The formatter must simply be passed to the logging handler.
json_handler = FileHandler("foo.json")
json_formatter = JsonFormatter({"level": "levelname",
"message": "message",
"loggerName": "name",
"processName": "processName",
"processID": "process",
"threadName": "threadName",
"threadID": "thread",
"timestamp": "asctime"})
json_handler.setFormatter(json_formatter)
Explanation :
While the logging.Formatter takes a string which it interpolates to output the formatted log record, the JsonFormatter takes a dictionary where the key will be the key of the logged value in the JSON string and the value is a string corresponding to an attribute of the LogRecord that can be logged.
(List available in the docs here)
Main "problem" would be parsing dates and timestamps, and the default formatter implementation has these class attributes, default_time_format and default_msec_format.
default_msec_format gets passed to time.strftime() and default_msec_format is interpolated to append miliseconds as time.strftime() doesn't provide formatting options for those.
The principle is that those are now instance attributes which can be provided in the form of time_format and msec_format to customize how the parent's class (unchanged, as it's not overwritten) formatTime() method behaves.
You could technically override it if you want to customize time formatting, but I personally found that using something else would either be redundant or limit the actual formatting options. But feel free to adjust as to your needs.
Output:
An example JSON record logged by the formatting options above, with the default time formatting options set in the class, would be:
{"level": "INFO", "message": "Starting service...", "loggerName": "root", "processName": "MainProcess", "processID": 25103, "threadName": "MainThread", "threadID": 4721200640, "timestamp": "2021-12-04T08:25:07.610Z"}
So based on #abarnert, i found this Link which provided a good path to making this concept work for the most part. The code as it stands is:
logger=logging.getLogger()
logger.setLevel(logging.DEBUG)
file_handler=logging.FileHandler('foo.log')
stream_handler=logging.StreamHandler()
stream_formatter=logging.Formatter(
'%(asctime)-15s %(levelname)-8s %(message)s')
file_formatter=logging.Formatter(
"{'time':'%(asctime)s', 'name': '%(name)s', \
'level': '%(levelname)s', 'message': '%(message)s'}"
)
file_handler.setFormatter(file_formatter)
stream_handler.setFormatter(stream_formatter)
logger.addHandler(file_handler)
logger.addHandler(stream_handler)
Although it does not fully meet the requirement, it doesnt require any pre processing, and allows me to create two log handlers.
Afterwards, i can use something like:
with open('foo.log') as f:
logs = f.read().splitlines()
for l in logs:
for key, value in eval(l):
do something ...
to pull dict objects instead of fighting with improperly formatted JSON to accomplish what i had set out to accomplish.
Still am hoping for a more elegant solution.
I wanted to have JSON output, so that I could handle it nicer in Promtail and Loki. I just updated the formatter in my logging.json, which I use as a dictConfig
"formatters": {
"normalFormatter": {
"format": "{\"time\": \"%(asctime)s\", \"name\": \"[%(name)s]\", \"levelname\": \"%(levelname)s\", \"message\": \"%(message)s\"}"
}
}
Load the config and get the root logger like:
import json
import logging
# setup logger
with open("logging.json") as f:
config_dict = json.load(f)
logging.config.dictConfig(config_dict)
# get root logger
logger = logging.getLogger(__name__)
If you want to use dictConfig, please make sure you gave all necessary fields.
https://docs.python.org/3/library/logging.config.html
You may want to define a formatter, a handler (which is using the formatter) and the logger (which is using the handler)
e.g. logging.json
{
"version": 1,
"disable_existing_loggers": false,
"formatters": {
"normalFormatter": {
"format": "{\"time\": \"%(asctime)s\", \"name\": \"[%(name)s]\", \"levelname\": \"%(levelname)s\", \"message\": \"%(message)s\"}"
}
},
"handlers": {
"demohandler": {
"level": "INFO",
"formatter": "normalFormatter",
"class": "logging.handlers.TimedRotatingFileHandler",
"filename": "./files/logs/YourLogFile.log",
"when": "d",
"interval": 30,
"backupCount": 4,
"utc": true
}
},
"loggers": {
"root": {
"handlers": ["demohandler"],
"level": "INFO"
},
"someModule": {
"handlers": ["demohandler"],
"level": "INFO",
"propagate": 0
}
}
}
With this code you can add the full traceback, timestamp and level to a json file of choice.
import json
import traceback
from datetime import datetime
def addLogging(logDict:dict):
loggingsFile = 'loggings.json'
with open(loggingsFile) as f:
data = json.load(f)
data.append(logDict)
with open(loggingsFile, 'w') as f:
json.dump(data, f)
def currentTimeUTC():
return datetime.now().strftime('%d/%m/%Y %H:%M:%S')
try:
print(5 / 0)
except ZeroDivisionError:
fullTraceback = str(traceback.format_exc())
addLogging({'timestamp': currentTimeUTC(), 'level': 'error', 'traceback': fullTraceback})
Output:
[
{
"timestamp": "09/06/2020 17:38:00",
"level": "error",
"traceback": "Traceback (most recent call last):\n File \"d:testFullTraceback.py\", line 19, in <module>\n print(5/0)\nZeroDivisionError: division by zero\n"
}
]
I could achieve this result using this custom formatter:
import json
import logging
class CustomJsonFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
super(CustomJsonFormatter, self).format(record)
output = {k: str(v) for k, v in record.__dict__.items()}
return json.dumps(output)
cf = CustomJsonFormatter()
sh = logging.StreamHandler()
sh.setFormatter(cf)
logger = logging.getLogger("my.module")
logger.addHandler(sh)
# simple json output
logger.warning("This is a great %s!", "log")
# enrich json output
logger.warning("This is an even greater %s!", "log", extra={'foo': 'bar'})
Output:
{"name": "my.module", "msg": "This is a great %s!", "args": "('log',)", "levelname": "WARNING", "levelno": "30", "pathname": "/Users/olivier/test.py", "filename": "test.py", "module": "test", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "20", "funcName": "<module>", "created": "1661868378.5048351", "msecs": "504.8351287841797", "relativeCreated": "1.3060569763183594", "thread": "4640826880", "threadName": "MainThread", "processName": "MainProcess", "process": "81360", "message": "This is a great log!"}
{"name": "my.module", "msg": "This is an even greater %s!", "args": "('log',)", "levelname": "WARNING", "levelno": "30", "pathname": "/Users/olivier/test.py", "filename": "test.py", "module": "test", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "22", "funcName": "<module>", "created": "1661868378.504962", "msecs": "504.9619674682617", "relativeCreated": "1.4328956604003906", "thread": "4640826880", "threadName": "MainThread", "processName": "MainProcess", "process": "81360", "foo": "bar", "message": "This is an even greater log!"}
If you don't mind pip installing a module to help, there is the json_log_formatter module. The json output does have more attributes than requested. The repo mentions customizing the attributes outputted, but I have not yet integrated that into a working example just yet.
json_log_formatter
import logging
import json_log_formatter
# Set Basic Logging
self.loggers = logging.getLogger(__name__)
self.loggers.setLevel(logging.DEBUG)
self.formatter = logging.Formatter(fmt='%(asctime)-15s %(levelname)-8s %(message)s', datefmt = '%a, %d %b %Y %H:%M:%S')
# Config for JSON File Handler
self.logFileHandler = logging.FileHandler(SOME-PATH, mode='a')
self.fileFormatter = json_log_formatter.VerboseJSONFormatter()
self.logFileHandler.setFormatter(self.fileFormatter)
self.logFileHandler.setLevel(logging.INFO)
self.loggers.addHandler(self.logFileHandler)
# Config for Stream Handler
self.logStreamHandler = logging.StreamHandler()
self.logStreamHandler.setFormatter(self.formatter)
self.logStreamHandler.setLevel(logging.INFO)
self.loggers.addHandler(self.logStreamHandler)
I wanted to save full LogRecord object, so I can later inspect my log with maximum of integration with module. So I inspected object like that:
class Handler_json(Handler):
def emit(self, record: LogRecord) -> None:
json_data = {}
for attr in filter(lambda attr: not attr.endswith("__"), dir(record)):
json_data[attr] = record.__getattribute__(attr)
del json_data["getMessage"]
print(json_data)
this is subclass of Handler, emit is rewritten method that is been called with every LogRecord. Dir returns all attributes and methods of the object. I am excluding special methods, and also deleting getMessage method, whitch is not needed for json object representation.
This can be nicely integrated in logging like that:
logger = getLogger(__name__)
logger.setLevel(DEBUG)
handle_json = Handler_json()
logger.addHandler(handle_json)
logger.info("my info")
result looks like this:
{
'args': (),
'created': 1639925351.0648422,
'exc_info': None,
'exc_text': None,
'filename': 'my_logging.py',
'funcName': 'restore_log_from_disk',
'levelname': 'INFO',
'levelno': 20,
'lineno': 142,
'module': 'my_logging',
'msecs': 64.84222412109375,
'msg': 'my info',
'name': '__main__',
'pathname':
'/home/jindrich/PycharmProjects/my_logging.py',
'process': 146331,
'processName': 'MainProcess',
'relativeCreated': 1.6417503356933594,
'stack_info': None,
'thread': 140347192436544,
'threadName': 'MainThread'
}
then you can load data from disk and recreat objects after some digging in doc.
I was able to create it using python-json-logger lib, it was simple and really easily to use.
Django
from pythonjsonlogger import jsonlogger
##This is to add custom keys
class CustomJsonFormatter(jsonlogger.JsonFormatter):
def add_fields(self, log_record, record, message_dict):
super(CustomJsonFormatter, self).add_fields(log_record, record, message_dict)
log_record['level'] = record.levelname
# Logging
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'json': {
'()': CustomJsonFormatter, # if you want to use custom logs class defined above
# '()': jsonlogger.JsonFormatter, # without custom logs
'format': '%(level)s %(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s %(threadName)s %(special)s %(run)s\
%(name)s %(created)s %(processName)s %(relativeCreated)d %(funcName)s %(levelno)d %(msecs)d %(pathname)s %(lineno)d %(filename)s'
},
},
'handlers': {
'null': {
'class': 'logging.NullHandler',
},
'console': {
'level': 'DEBUG',
'class': 'logging.StreamHandler',
'formatter': 'json'
},
},
.....
}
Flask
from logging.config import dictConfig
from pythonjsonlogger import jsonlogger
import os
# This will set global root logging config across all the modules using in the app
##This is to add custom keys
class CustomJsonFormatter(jsonlogger.JsonFormatter):
def add_fields(self, log_record, record, message_dict):
super(CustomJsonFormatter, self).add_fields(log_record, record, message_dict)
log_record['level'] = record.levelname
def setup():
LOG_FILE = '/tmp/app/app.json'
if not os.path.exists(os.path.dirname(LOG_FILE)): # if LOG_FILE dir doesn't exist, creates it.
os.makedirs(os.path.dirname(LOG_FILE))
dictConfig({
'version': 1,
'formatters': {
'default': {
'format': '%(asctime)s - %(module)s - %(levelname)s - %(message)s',
},
'json': {
'()': CustomJsonFormatter, # if you want to use custom logs class defined above
# '()': jsonlogger.JsonFormatter, # without custom logs
'format': '%(level)s %(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s %(threadName)s %(special)s %(run)s\
%(name)s %(created)s %(processName)s %(relativeCreated)d %(funcName)s %(levelno)d %(msecs)d %(pathname)s %(lineno)d %(filename)s'
},
},
'handlers': {'file': {
'class': 'logging.handlers.RotatingFileHandler',
'filename': LOG_FILE,
'maxBytes': 10485760,
'backupCount': 5,
'formatter': 'json'
},
'console':{
'class':'logging.StreamHandler',
'formatter': 'json'
}},
'root': {
'level': 'INFO',
'handlers': ['file', 'console']
}
})
Hope this helps easy way setup.
I am trying to extend RotatingFile Handler to say FooBar.
class FooBar(RotatingFileHandler) :
def __init__(self, filename, mode='a', maxBytes=0,backupCount=0, encoding=None, delay=0) :
RotatingHandler.__init__(self, filename, mode, maxBytes, backupCount, encoding, delay)
I configure it using
LOGGING = {
'version': 1,
'disable_existing_loggers': True,
'formatters': {
'verbose': {
'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
},
'simple': {
'format': '%(levelname)s %(asctime)s %(message)s'
},
},
'handlers': {
'file':{
'level': 'ERROR',
'class': 'myhandler.FooBar',
'formatter': 'simple',
'filename': '/tmp/cattle.txt',
'mode': 'a',
'maxBytes': 16,
'backupCount' : 100,
},
#-- Remaining part truncated ###
logging.config.dictConfig(LOGGING) ### === ERROR here
When I use it ; I get an error
File "/usr/lib/python2.7/logging/config.py", line 576, in configure
'%r: %s' % (name, e))
ValueError: Unable to configure handler 'file': global name 'RotatingHandler' is not defined
RotatingHandler is not in scope, so you would need something like this to bring it into scope:
from logging.handlers import RotatingFileHandler
However, have a look at this example:
How to log everything into a file using RotatingFileHandler by using logging.conf file?
You may not need to create your own class to accomplish what you want to do.