We Keep Coding

Python logging with {} format using logging.config.dictConfig

I'm trying to use the string.format() style with python logging module. I copied some examples from this site and modified it with the the new formatter. I'd like to use logging.config.dictConfig to specify the logging format.
My script is:
import sys, logging, logging.config
DEFAULT_LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'standard': {
'format': '{message}',
'style': '{' # use string.format()
},
},
'handlers': {
'default': {
'level': 'INFO',
'formatter': 'standard',
'class': 'logging.StreamHandler',
'stream': 'ext://sys.stdout', # Default is stderr
},
},
'loggers': {
'__main__': { # if __name__ == '__main__'
'handlers': ['default'],
'level': 'DEBUG',
'formatter': 'standard'
},
'': {
'handlers': ['default'],
'level': 'INFO',
'formatter': 'standard'
},
}
}
logger = logging.getLogger(__name__)
logging.config.dictConfig(DEFAULT_LOGGING)
if __name__ == '__main__':
logger.info('Hello, {}', 'log')
sys.exit(0)
But an exception raised:
--- Logging error --- Traceback (most recent call last): File "/usr/lib/python3.8/logging/init.py", line 1081, in emit
msg = self.format(record) File "/usr/lib/python3.8/logging/init.py", line 925, in format
return fmt.format(record) File "/usr/lib/python3.8/logging/init.py", line 664, in format
record.message = record.getMessage() File "/usr/lib/python3.8/logging/init.py", line 369, in getMessage
msg = msg % self.args TypeError: not all arguments converted during string formatting Call stack: File "test/dbg/log.py", line
48, in
logger.info('Hello, {}', 'log') Message: 'Hello, {}' Arguments: ('log',)
For some reason python is trying to format using % formatter (msg = msg % self.args). How can I modify my script to point to the correct formatting? Any help will be appreciated.
Thanks in advance.
-Uri

One way of addressing this issue is to create a thin adapter around the built-in logging formatter(s).
Example:
import sys, logging, logging.config
class standardFormatterFactory(logging.Formatter):
def __init__(self, fmt, datefmt, style, validate=False):
try:
# 3.x
super(standardFormatterFactory, self).__init__(fmt=fmt, datefmt=datefmt, style=style, validate=validate)
except:
# 2.7
super(standardFormatterFactory, self).__init__(fmt=fmt, datefmt=datefmt)
self._usesTime = self._fmt.find('%(asctime)') >= 0 or self._fmt.find('{asctime}') >= 0
def usesTime(self):
'''Override logging.Formatter.usesTime()'''
return self._usesTime
def format(self, record):
'''Override logging.Formatter.format()'''
try:
return super(standardFormatterFactory, self).format(record=record)
except TypeError as ex:
# message contains {} formate specifiers
record.message = record.msg.format(*record.args)
if self.usesTime():
record.asctime = self.formatTime(record, self.datefmt)
return self._fmt.format(**record.__dict__)
DEFAULT_LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'propagate': False,
'formatters': {
'standard': {
'()' : standardFormatterFactory,
'fmt': '{asctime}-{levelname}: {message}',
'datefmt': '%Y-%m-%d %H:%M:%S',
'style': '{'
},
},
'handlers': {
'default': {
'level': 'DEBUG',
'formatter': 'standard',
'class': 'logging.StreamHandler',
'stream': 'ext://sys.stdout', # Default is stderr
},
},
'loggers': {
'__main__': { # if __name__ == '__main__'
'handlers': ['default'],
'level': 'DEBUG',
'formatter': 'standard'
},
}
}
logger = logging.getLogger(__name__)
logging.config.dictConfig(DEFAULT_LOGGING)
if __name__ == '__main__':
logger.info('Hello, {}', 'log')
sys.exit(0)

python: How to start and stop a logger whenever i want

I am trying to log sql statements in a code in my Django Application
Currently i am using the following logger config in my settings.py
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'sql': {
'()': SQLFormatter,
'format': '[%(duration).3f] %(statement)s',
},
'verbose': {
'format': '%(levelname)s %(funcName)s() %(pathname)s[:%(lineno)s] %(name)s \n%(message)s'
}
},
'handlers': {
'console': {
'level': 'DEBUG',
'formatter': 'verbose',
'class': 'logging.StreamHandler',
},
'sql': {
'class': 'logging.StreamHandler',
'formatter': 'sql',
'level': 'DEBUG',
}
}
}
In genereal to log sql in django we can add the django.db.backends to the logger.config in the settings.py
'loggers': {
'django.db.backends': {
'handlers': ['sql'],
'level': 'DEBUG',
'propagate': False,
},
But the problem is it will log every sql statement. So how can we start and stop logging for django.db.backends in between code.
I have the following code in my views.py
def someview(request)
# start logging from here
user_set = User.objects.all()
for user in user_set:
print(user.last_name)
# stop logging from here
Also I want to use the sql handler which I defined in the logging config.
What code will go in start and stop logging place in the above view function.

Create a filter class and add an instance to the logger or handler.
class LoggerGate:
def __init__(self, state='open'):
self.state = state
def open(self):
self.state = 'open'
def close(self):
self.state = 'closed'
def filter(self, record):
return self.state == 'open'
Create a filter, initialized in the 'closed' state.
Get the 'django.db.backends' logger and add the filter.
gate = LoggerGate('closed')
sql_logger = logging.getLogger('django.db.backends')
sql_logger.addFilter(gate)
Then call the open or close method to limit logging to where you want it.
def someview(request)
gate.open() # start logging from here
user_set = User.objects.all()
for user in user_set:
print(user.last_name)
gate.close() # stop logging here

Just summarizing from the above answer and also from the answer of Gabriel C, which both are same and also from the answer of Sraw
My goal was to log sql using django django.db.backends. But the problem with it is that it will log all the sqls. I want to log only sqls in a particular section of a code or whereever i want to see the sqls. So the following way i could do it.
logging config inside settings.py:
# Filter class to stop or start logging for "django.db.backends"
class LoggerGate:
def __init__(self, state='closed'):
# We found that the settings.py runs twice and the filters are created twice. So we have to keep only one. So we delete all the previous filters before we create the new one
import logging
logger_database = logging.getLogger("django.db.backends")
try:
for filter in logger_database.filters:
logger_database.removeFilter(filter)
except Exception as e:
pass
self.state = state
def open(self):
self.state = 'open'
def close(self):
self.state = 'closed'
def filter(self, record):
"""
Determine if the specified record is to be logged.
Is the specified record to be logged? Returns 0/False for no, nonzero/True for
yes. If deemed appropriate, the record may be modified in-place.
"""
return self.state == 'open'
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'handlers': {
'sql': {
'class': 'logging.StreamHandler',
'level': 'DEBUG',
}
},
'filters': {
'myfilter': {
'()': LoggerGate,
}
},
'loggers': {
'django.db.backends': {
'handlers': ['sql'],
'level': 'DEBUG',
'propagate': False,
'filters': ['myfilter']
}
}
}
Then in the views.py
import logging
logger = logging.getLogger(__name__)
logger_database = logging.getLogger("django.db.backends")
def test1(request):
logger_database.filters[0].open()
#Will allow priting of sql satatements from here
from django import db
user_set = User.objects.all()
for user in user_set: # Here sql is executed and is printed to console
pass
#Will stop priting of sql satatements after this
logger_database.filters[0].close()
from django import db
user_set = User.objects.all()
for user in user_set: # Here sql is executed and is not printed to console
pass
now = datetime.datetime.now()
html = "<html><body>Internal purpose</body></html>"
return HttpResponse(html)
If one wants to print the sql in formatted and colorful way use this in the settings.py
# SQL formatter to be used for the handler used in logging "django.db.backends"
class SQLFormatter(logging.Formatter):
def format(self, record):
# Check if Pygments is available for coloring
try:
import pygments
from pygments.lexers import SqlLexer
from pygments.formatters import TerminalTrueColorFormatter
except ImportError:
pygments = None
# Check if sqlparse is available for indentation
try:
import sqlparse
except ImportError:
sqlparse = None
# Remove leading and trailing whitespaces
sql = record.sql.strip()
if sqlparse:
# Indent the SQL query
sql = sqlparse.format(sql, reindent=True)
if pygments:
# Highlight the SQL query
sql = pygments.highlight(
sql,
SqlLexer(),
#TerminalTrueColorFormatter(style='monokai')
TerminalTrueColorFormatter()
)
# Set the record's statement to the formatted query
record.statement = sql
return super(SQLFormatter, self).format(record)
# Filter class to stop or start logging for "django.db.backends"
class LoggerGate:
def __init__(self, state='closed'):
# We found that the settings.py runs twice and the filters are created twice. So we have to keep only one. So we delete all the previous filters before we create the new one
import logging
logger_database = logging.getLogger("django.db.backends")
try:
for filter in logger_database.filters:
logger_database.removeFilter(filter)
except Exception as e:
pass
self.state = state
def open(self):
self.state = 'open'
def close(self):
self.state = 'closed'
def filter(self, record):
"""
Determine if the specified record is to be logged.
Is the specified record to be logged? Returns 0/False for no, nonzero/True for
yes. If deemed appropriate, the record may be modified in-place.
"""
return self.state == 'open'
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'sql': {
'()': SQLFormatter,
'format': '[%(duration).3f] %(statement)s',
}
},
'handlers': {
'sql': {
'class': 'logging.StreamHandler',
'formatter': 'sql',
'level': 'DEBUG',
}
},
'filters': {
'myfilter': {
'()': LoggerGate,
}
},
'loggers': {
'django.db.backends': {
'handlers': ['sql'],
'level': 'DEBUG',
'propagate': False,
'filters': ['myfilter']
}
}
}

django database query log linenumber

I am logging my database queries in Django along with the pathname and linenumber.
Right now i am getting these logs:
07/Dec/2018 14:25:00 DEBUG django.db.backends utils **/Users/XXXXX/.idea/lib/python2.7/site-packages/django/db/backends/utils.py:89**
(0.340) SELECT "metadata"."metaname", "metadata"."description", "metadata"."attributes" FROM "metadata" WHERE "metadata"."metaname" = 'date_type'; args=('date_type',)
For all queries, I am getting the same path and line number. Is there any way I can capture the line number from my main application instead of the one from utils.
Current logging Implementation:
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'handlers': {
'console': {
'level': 'DEBUG',
'class': 'logging.StreamHandler',
'formatter': 'color'
},
},
'loggers': {
'django.db.backends': {
'handlers': ['console'],
'level': 'DEBUG',
'propogate': True,
}
}
}
Using python 2.7 and django 1.9

The slightly optimized version from User #will-keeling
Logging configuration for Django for output line numbers for each db request.
Note: If you want to use it for tests you need to set DEBUG=True for tests How do you set DEBUG to True when running a Django test?
import logging
import traceback
from django.conf import settings
class StackInfoHandler(logging.StreamHandler):
trim = 5
def emit(self, record):
super(StackInfoHandler, self).emit(record)
trace = traceback.format_stack()
stack1 = [str(row) for row in trace]
stack2 = [s for s in stack1 if settings.BASE_DIR in s and 'format_stack' not in s]
stack3 = [s for s in stack2 if 'test' not in s]
if not stack3:
stack3 = stack2 # include test call
if stack3:
stack4 = ''.join(stack3[-self.trim:]) # take only last records
stack5 = f"Stack {self.terminator} {''.join(stack4)}"
self.stream.write(stack5)
self.stream.write(self.terminator)
self.flush()
Logging Config (partitial)
LOGGING = {
'handlers': {
'db-console': {
'level': 'DEBUG',
'class': 'settings.local.StackInfoHandler', # Reference the custom handler
'formatter': 'simple',
},
'loggers': {
'django.db.backends': {
'handlers': ['db-console'],
'level': 'DEBUG',
'propagate': False
},
}
}
}
This will show you only stack trace from your Django codebase like below
[2020-05-25 17:49:17,977]: (0.000) INSERT INTO `contacts_contactscount` (`user_id`, `date`, `amount`) VALUES (338, '2020-05-25 17:49:17', 7); args=[338, '2020-05-25 17:49:17', 7]
Stack
File "<project-root>/api/views/contacts.py", line 164, in create
Contact.objects.filter(pk__in=to_delete).delete()
File "<project-root>/<folder>/contacts/models.py", line 54, in delete
create_deletion_log.delay(obj, deleted_timestamp)
File "<project-root>/<folder>/contacts/tasks.py", line 31, in create_deletion_log
contact.save()
File "<project-root>/<folder>/contacts/models.py", line 118, in save
Contact.objects.contacts_added_hook(self.user)
File "<project-root>/<folder>/contacts/models.py", line 67, in contacts_added_hook
current_total = user.profile.contacts_total
File "<project-root>/<folder>/profile/models.py", line 631, in contacts_total
ContactsCount.objects.create(user=self.user, amount=count)

I'm guessing that you're trying to determine which lines in your application are responsible for running which queries.
One way to achieve this would be to create a custom handler that prints out the current stack at the point where Django logs the query. That would allow you to see which line in your application is executing.
You could create a custom handler such as:
import logging
import traceback
class StackInfoHandler(logging.StreamHandler):
trim = 5
def emit(self, record):
super(StackInfoHandler, self).emit(record)
stack = ''.join(
str(row) for row in traceback.format_stack()[:-self.trim]
)
self.stream.write(stack)
And then in your logging config, you could just switch the handler class to use the StackInfoHandler:
'handlers': {
'console': {
'level': 'DEBUG',
'class': 'my.package.StackInfoHandler', # Reference the custom handler
'formatter': 'color'
},
},
Note that the StackInfoHandler trims 5 lines off the stack so that it doesn't show you stack frames from the logging framework itself. You might need to tweak this number (5 works for me locally).

How to set logger class universally in Django App

I have a logger that just adds log_line if none is provided when calling logger.debug
class UrlLogger(logging.Logger):
def _log(self, level, msg, args, exc_info=None, extra=None):
if extra is None:
extra = {'log_line':' '}
super(UrlLogger, self)._log(level, msg, args, exc_info, extra)
I have about 20 different modules that I need to go into and add
logging.setLoggerClass(UrlLogger)
How can I set this as default in the settings?
My settings currently look like this:
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'verbose': {
'format': '%(levelname)s %(name)s %(asctime)s %(module)s %(process)d %(thread)d %(pathname)s#%(lineno)s: %(message)s'
},
'simple': {
'format': '[%(levelname)8s] [%(asctime)s] %(module)10s/%(filename)s:%(log_line)s - %(message)s',
'datefmt': '%d-%m-%Y %H:%M:%S'
},
},
'handlers': {
'console': {
'class': 'logging.StreamHandler',
'level': LOG_LEVEL,
'formatter': 'simple'
},
},
'loggers': {
'': {
'level': LOG_LEVEL,
'handlers': ['console'],
'class': ['UrlLogger'],
},
As you can see, I tried setting it here, but I don't think that's working because I am still getting errors.
Message: 'Not Found: %s'
Arguments: ('/tool_job_list/1/1fa119372de6/job/check/jexecution/report',)
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.6/logging/__init__.py", line 992, in emit
msg = self.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 838, in format
return fmt.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 578, in format
s = self.formatMessage(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 547, in formatMessage
return self._style.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 391, in format
return self._fmt % record.__dict__
KeyError: 'log_line'
This error occurs when the front end makes failed API calls (wrong uri) to the django backend. I'm thinking that the failed api call is logging the error without using my UrlLogger (otherwise there would not be a key error unless there is an error in my Logger Class).
What am I doing wrong here?

How do I add log handler in celery?

So I have my tasks.py here is an exerpt:
import builtins
import logging
import os
import urllib
import inspect
from celery import Celery
from common.environment_helper import EnvironmentHelper
from config import log
# Logging functionality
logger = logging.getLogger(__name__)
EnvironmentHelper.set_environment(logger)
app = Celery('tasks', broker=BROKER__URL, broker_transport_options=BROKER_TRANSPORT_OPTIONS)
app.conf.CELERY_ACCEPT_CONTENT = ['json']
app.conf.CELERY_TASK_SERIALIZER = 'json'
app.conf.CELERYD_PREFETCH_MULTIPLIER = 1
app.log.already_setup=True
app.conf.CELERY_ENABLE_REMOTE_CONTROL = False
app.conf.CELERY_DEFAULT_QUEUE = queue_name
#app.task
def convert_file(file_conversion_json):
file_conversion_json_copy = file_conversion_json.copy()
So basically I want to take a value from the dictionary and add it to my log. I have done this successfully in my actual application by using the following code:
import uuid
import logging
import flask
# Generate a new request ID, optionally including an original request ID
def generate_request_id(original_id):
new_id = uuid.uuid4()
if not original_id:
return new_id
else:
new_id = original_id
return new_id
# Returns the current request ID or a new one if there is none
# In order of preference:
# * If we've already created a request ID and stored it in the flask.g context local, use that
# * If a client has passed in the X-Request-Id header, create a new ID with that prepended
# * Otherwise, generate a request ID and store it in flask.g.request_id
def request_id():
if getattr(flask.g, 'request_id', None):
return flask.g.request_id
headers = flask.request.headers
original_request_id = headers.get("X-Request-Id")
new_uuid = generate_request_id(original_request_id)
flask.g.request_id = new_uuid
return new_uuid
class RequestIdFilter(logging.Filter):
# This is a logging filter that makes the request ID available for use in
# the logging format. Note that we're checking if we're in a request
# context, as we may want to log things before Flask is fully loaded.
def filter(self, record):
record.request_id = request_id() if flask.has_request_context() else ''
return True
............
# log.py
import logging.config
import os
LOGGER_CONFIGURATION = {
'version': 1,
'filters': {
'request_id': {
'()': 'config.utils.RequestIdFilter',
},
},
'formatters': {
'standard': {
'format': '%(asctime)s - %(name)s.%(module)s.%(funcName)s:%(lineno)d - %(levelname)s - %(request_id)s - %(message)s',
},
},
'handlers': {
'console': {
'class': 'logging.StreamHandler',
'level': 'INFO',
'filters': ['request_id'],
'formatter': 'standard'
}
},
'loggers': {
'': {
'handlers': ['console'],
'level':'INFO',
},
'app': {
'handlers': ['console'],
'level':'INFO',
},
}
}
logging.config.dictConfig(LOGGER_CONFIGURATION)
But it doesn't work for celery and I have no idea how to add a variable on the fly like that. Any advice?

So I figured out that if I add this to my task:
logFormatter = logging.Formatter('%(asctime)s - %(name)s.%(module)s.%(funcName)s:%(lineno)d - %(levelname)s - Request ID:'+ file_conversion_json['x_request_id'] + ' - %(message)s')
rootLogger = logging.getLogger()
consoleHandler = logging.StreamHandler()
consoleHandler.setFormatter(logFormatter)
rootLogger.addHandler(consoleHandler)
It will add what I want to the log output, but now it duplicated my log messages, I am getting each message twice in the log output....