We Keep Coding

python: logging: can we added multiple filters to the logger and which one is considered

I am trying to understand how multiple Filters (one defined in config and other in the code) in Python logging work.
I am working on a Django project and below is my logger config in settings.py
My goal is to switch on and switch off the logger whenever i want. So using filters i am trying to switch off the logger by returning False (0)
1) Switch off the logger in the starting
class StartFilter(object):
def filter(self, record):
"""
Determine if the specified record is to be logged.
Is the specified record to be logged? Returns 0 for no, nonzero for
yes. If deemed appropriate, the record may be modified in-place.
"""
return 0
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'verbose': {
'format': '%(levelname)s %(funcName)s() %(pathname)s[:%(lineno)s] %(name)s \n%(message)s'
}
},
'handlers': {
'console': {
'level': 'DEBUG',
'formatter': 'verbose',
'class': 'logging.StreamHandler',
},
},
'filters': {
'myfilter': {
'()': StartFilter,
}
},
'loggers': {
'log_testing': {
'handlers': ['console'],
'level': 'DEBUG',
'propagate': False,
'filters': ['myfilter']
},
}
}
I have added the filter to the logger. 'filters': ['myfilter']
2) Switch on and off the logger in the views.py file where i want the logging to be seen
# to switch on logger
class LoggerGateStart(object):
def filter(self, record):
"""
Determine if the specified record is to be logged.
Is the specified record to be logged? Returns 0 for no, nonzero for
yes. If deemed appropriate, the record may be modified in-place.
"""
return 1
# to switch off logger
class LoggerGateStop(object):
def filter(self, record):
"""
Determine if the specified record is to be logged.
Is the specified record to be logged? Returns 0 for no, nonzero for
yes. If deemed appropriate, the record may be modified in-place.
"""
return 0
import logging
logger = logging.getLogger("log_testing")
...
logging.debug("Some text Before) # i dont want this to be logged
...
gatestart = LoggerGateStart()
logger_database.addFilter(gatestart)
...
logging.debug("Some text) # i want this to be logged
...
gatestop = LoggerGateStop()
logger_database.addFilter(gatestop)
...
logging.debug("Some text after") # i dont want this to be logged even
if it exist
...
I found its not working this way. It considers the StartFilter only and not consider LoggerGateStart or LoggerGateStop and does not print any log to the console
How can i do this
ANSWER I USED BASED ON THE ANSWER OF Gabriel C
My goal was to log sql using django django.db.backends. But the problem with it is that it will log all the sqls. I want to log only sqls in a particular section of a code or whereever i want to see the sqls. So the following way i could do it.
logging config inside settings.py:
# Filter class to stop or start logging for "django.db.backends"
class LoggerGate:
def __init__(self, state='closed'):
# We found that the settings.py runs twice and the filters are created twice. So we have to keep only one. So we delete all the previous filters before we create the new one
import logging
logger_database = logging.getLogger("django.db.backends")
try:
for filter in logger_database.filters:
logger_database.removeFilter(filter)
except Exception as e:
pass
self.state = state
def open(self):
self.state = 'open'
def close(self):
self.state = 'closed'
def filter(self, record):
"""
Determine if the specified record is to be logged.
Is the specified record to be logged? Returns 0/False for no, nonzero/True for
yes. If deemed appropriate, the record may be modified in-place.
"""
return self.state == 'open'
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'handlers': {
'sql': {
'class': 'logging.StreamHandler',
'level': 'DEBUG',
}
},
'filters': {
'myfilter': {
'()': LoggerGate,
}
},
'loggers': {
'django.db.backends': {
'handlers': ['sql'],
'level': 'DEBUG',
'propagate': False,
'filters': ['myfilter']
}
}
}
Then in the views.py
import logging
logger = logging.getLogger(__name__)
logger_database = logging.getLogger("django.db.backends")
def test1(request):
logger_database.filters[0].open()
#Will allow priting of sql satatements from here
from django import db
user_set = User.objects.all()
for user in user_set: # Here sql is executed and is printed to console
pass
#Will stop priting of sql satatements after this
logger_database.filters[0].close()
from django import db
user_set = User.objects.all()
for user in user_set: # Here sql is executed and is not printed to console
pass
now = datetime.datetime.now()
html = "<html><body>Internal purpose</body></html>"
return HttpResponse(html)
If one wants to print the sql in formatted and colorful way use this in the settings.py
# SQL formatter to be used for the handler used in logging "django.db.backends"
class SQLFormatter(logging.Formatter):
def format(self, record):
# Check if Pygments is available for coloring
try:
import pygments
from pygments.lexers import SqlLexer
from pygments.formatters import TerminalTrueColorFormatter
except ImportError:
pygments = None
# Check if sqlparse is available for indentation
try:
import sqlparse
except ImportError:
sqlparse = None
# Remove leading and trailing whitespaces
sql = record.sql.strip()
if sqlparse:
# Indent the SQL query
sql = sqlparse.format(sql, reindent=True)
if pygments:
# Highlight the SQL query
sql = pygments.highlight(
sql,
SqlLexer(),
#TerminalTrueColorFormatter(style='monokai')
TerminalTrueColorFormatter()
)
# Set the record's statement to the formatted query
record.statement = sql
return super(SQLFormatter, self).format(record)
# Filter class to stop or start logging for "django.db.backends"
class LoggerGate:
def __init__(self, state='closed'):
# We found that the settings.py runs twice and the filters are created twice. So we have to keep only one. So we delete all the previous filters before we create the new one
import logging
logger_database = logging.getLogger("django.db.backends")
try:
for filter in logger_database.filters:
logger_database.removeFilter(filter)
except Exception as e:
pass
self.state = state
def open(self):
self.state = 'open'
def close(self):
self.state = 'closed'
def filter(self, record):
"""
Determine if the specified record is to be logged.
Is the specified record to be logged? Returns 0/False for no, nonzero/True for
yes. If deemed appropriate, the record may be modified in-place.
"""
return self.state == 'open'
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'sql': {
'()': SQLFormatter,
'format': '[%(duration).3f] %(statement)s',
}
},
'handlers': {
'sql': {
'class': 'logging.StreamHandler',
'formatter': 'sql',
'level': 'DEBUG',
}
},
'filters': {
'myfilter': {
'()': LoggerGate,
}
},
'loggers': {
'django.db.backends': {
'handlers': ['sql'],
'level': 'DEBUG',
'propagate': False,
'filters': ['myfilter']
}
}
}

Create only one filter and use it's instance to control if logs should be accepted or not.
from logging import getLogger
class LoggerGate(object):
def __init__(self):
self.started = False
def start(self):
self.started = True
def stop(self):
self.started = False
def filter(self, record):
"""
Determine if the specified record is to be logged.
Returns True is this LoggerGate is started, False otherwise.
"""
return self.started
logger_database = getLogger("log_testing")
logger_gate = LoggerGate()
logger_database.addFilter(logger_gate)
logger_database.critical('this is not logged')
logger_gate.start()
logger_database.critical('this is logged')
logger_gate.stop()
logger_database.critical('this is not logged')

Gabriel C gives a great solution. And to explain more, filter works in sequence, which means the record is passed to each filter one by one. And will stop at the one returns a zero. So as your StartFilter returns 0, it will directly drop all records.

python: How to start and stop a logger whenever i want

I am trying to log sql statements in a code in my Django Application
Currently i am using the following logger config in my settings.py
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'sql': {
'()': SQLFormatter,
'format': '[%(duration).3f] %(statement)s',
},
'verbose': {
'format': '%(levelname)s %(funcName)s() %(pathname)s[:%(lineno)s] %(name)s \n%(message)s'
}
},
'handlers': {
'console': {
'level': 'DEBUG',
'formatter': 'verbose',
'class': 'logging.StreamHandler',
},
'sql': {
'class': 'logging.StreamHandler',
'formatter': 'sql',
'level': 'DEBUG',
}
}
}
In genereal to log sql in django we can add the django.db.backends to the logger.config in the settings.py
'loggers': {
'django.db.backends': {
'handlers': ['sql'],
'level': 'DEBUG',
'propagate': False,
},
But the problem is it will log every sql statement. So how can we start and stop logging for django.db.backends in between code.
I have the following code in my views.py
def someview(request)
# start logging from here
user_set = User.objects.all()
for user in user_set:
print(user.last_name)
# stop logging from here
Also I want to use the sql handler which I defined in the logging config.
What code will go in start and stop logging place in the above view function.

Create a filter class and add an instance to the logger or handler.
class LoggerGate:
def __init__(self, state='open'):
self.state = state
def open(self):
self.state = 'open'
def close(self):
self.state = 'closed'
def filter(self, record):
return self.state == 'open'
Create a filter, initialized in the 'closed' state.
Get the 'django.db.backends' logger and add the filter.
gate = LoggerGate('closed')
sql_logger = logging.getLogger('django.db.backends')
sql_logger.addFilter(gate)
Then call the open or close method to limit logging to where you want it.
def someview(request)
gate.open() # start logging from here
user_set = User.objects.all()
for user in user_set:
print(user.last_name)
gate.close() # stop logging here

Just summarizing from the above answer and also from the answer of Gabriel C, which both are same and also from the answer of Sraw
My goal was to log sql using django django.db.backends. But the problem with it is that it will log all the sqls. I want to log only sqls in a particular section of a code or whereever i want to see the sqls. So the following way i could do it.
logging config inside settings.py:
# Filter class to stop or start logging for "django.db.backends"
class LoggerGate:
def __init__(self, state='closed'):
# We found that the settings.py runs twice and the filters are created twice. So we have to keep only one. So we delete all the previous filters before we create the new one
import logging
logger_database = logging.getLogger("django.db.backends")
try:
for filter in logger_database.filters:
logger_database.removeFilter(filter)
except Exception as e:
pass
self.state = state
def open(self):
self.state = 'open'
def close(self):
self.state = 'closed'
def filter(self, record):
"""
Determine if the specified record is to be logged.
Is the specified record to be logged? Returns 0/False for no, nonzero/True for
yes. If deemed appropriate, the record may be modified in-place.
"""
return self.state == 'open'
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'handlers': {
'sql': {
'class': 'logging.StreamHandler',
'level': 'DEBUG',
}
},
'filters': {
'myfilter': {
'()': LoggerGate,
}
},
'loggers': {
'django.db.backends': {
'handlers': ['sql'],
'level': 'DEBUG',
'propagate': False,
'filters': ['myfilter']
}
}
}
Then in the views.py
import logging
logger = logging.getLogger(__name__)
logger_database = logging.getLogger("django.db.backends")
def test1(request):
logger_database.filters[0].open()
#Will allow priting of sql satatements from here
from django import db
user_set = User.objects.all()
for user in user_set: # Here sql is executed and is printed to console
pass
#Will stop priting of sql satatements after this
logger_database.filters[0].close()
from django import db
user_set = User.objects.all()
for user in user_set: # Here sql is executed and is not printed to console
pass
now = datetime.datetime.now()
html = "<html><body>Internal purpose</body></html>"
return HttpResponse(html)
If one wants to print the sql in formatted and colorful way use this in the settings.py
# SQL formatter to be used for the handler used in logging "django.db.backends"
class SQLFormatter(logging.Formatter):
def format(self, record):
# Check if Pygments is available for coloring
try:
import pygments
from pygments.lexers import SqlLexer
from pygments.formatters import TerminalTrueColorFormatter
except ImportError:
pygments = None
# Check if sqlparse is available for indentation
try:
import sqlparse
except ImportError:
sqlparse = None
# Remove leading and trailing whitespaces
sql = record.sql.strip()
if sqlparse:
# Indent the SQL query
sql = sqlparse.format(sql, reindent=True)
if pygments:
# Highlight the SQL query
sql = pygments.highlight(
sql,
SqlLexer(),
#TerminalTrueColorFormatter(style='monokai')
TerminalTrueColorFormatter()
)
# Set the record's statement to the formatted query
record.statement = sql
return super(SQLFormatter, self).format(record)
# Filter class to stop or start logging for "django.db.backends"
class LoggerGate:
def __init__(self, state='closed'):
# We found that the settings.py runs twice and the filters are created twice. So we have to keep only one. So we delete all the previous filters before we create the new one
import logging
logger_database = logging.getLogger("django.db.backends")
try:
for filter in logger_database.filters:
logger_database.removeFilter(filter)
except Exception as e:
pass
self.state = state
def open(self):
self.state = 'open'
def close(self):
self.state = 'closed'
def filter(self, record):
"""
Determine if the specified record is to be logged.
Is the specified record to be logged? Returns 0/False for no, nonzero/True for
yes. If deemed appropriate, the record may be modified in-place.
"""
return self.state == 'open'
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'sql': {
'()': SQLFormatter,
'format': '[%(duration).3f] %(statement)s',
}
},
'handlers': {
'sql': {
'class': 'logging.StreamHandler',
'formatter': 'sql',
'level': 'DEBUG',
}
},
'filters': {
'myfilter': {
'()': LoggerGate,
}
},
'loggers': {
'django.db.backends': {
'handlers': ['sql'],
'level': 'DEBUG',
'propagate': False,
'filters': ['myfilter']
}
}
}

logs inside django channels consumers not showing up

Environment:
OS: redhat
python version: 3.6
django: 2.1
django channels: 2.1.3
The following is my logging.conf file (same level as settings.py):
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'standard': {
'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s',
'datefmt': '%Y-%m-%d %H:%M:%S',
},
},
'handlers': {
'file': {
'level': 'DEBUG',
'class': 'logging.FileHandler',
'filename': 'debug.log',
'formatter': 'standard'
},
},
'loggers': {
'django': {
'handlers': ['file'],
'level': 'DEBUG',
'propagate': True,
},
},
}
django_logger = logging.getLogger('django')
and the function to log as follows:
def log_it(*args):
try:
django_logger.info(str(args).encode('utf-8'))
return True
except:
django_logger.info('#### Exception in LOGGING!!!!!!')
return False
I am using this to log inside django consumers as follows:
class EchoConsumer(SyncConsumer):
def websocket_connect(self, event):
try:
log_it('inside EchoConsumer connect()', event)
self.send({
"type": "websocket.accept",
})
return True
except:
error = traceback.format_exc()
write_error_log(error)
return False
def websocket_receive(self, event):
try:
log_it('inside EchoConsumer receive()', event)
self.send({
"type": "websocket.send",
"text": event["text"],
})
return True
except:
error = traceback.format_exc()
write_error_log(error)
return False
The log_it() used inside the consumers aren't being invoked.
(Assumption: Integrating channels with django must have overriden the django logging settings)
I checked if there was any problem in reaching the consumers in the daphne access logs.
I also checked my nginx conf, but the connection requests were recorded as follows:
127.0.0.1:45100 - - [08/Oct/2018:19:00:58] "WSCONNECTING /ws/event/" - -
127.0.0.1:45100 - - [08/Oct/2018:19:00:58] "WSCONNECT /ws/event/" - -
Note: I have linked /ws/event to EchoConsumer in routing.py
My expectation is:
127.0.0.1:45100 - - [08/Oct/2018:19:00:58] "WSCONNECTING /ws/event/" - -
127.0.0.1:45100 - - [08/Oct/2018:19:00:58] "WSCONNECT /ws/event/" - -
inside EchoConsumer connect() ....
Any idea why the log statements are not being displayed?
P.S related issue on github

How do I add log handler in celery?

So I have my tasks.py here is an exerpt:
import builtins
import logging
import os
import urllib
import inspect
from celery import Celery
from common.environment_helper import EnvironmentHelper
from config import log
# Logging functionality
logger = logging.getLogger(__name__)
EnvironmentHelper.set_environment(logger)
app = Celery('tasks', broker=BROKER__URL, broker_transport_options=BROKER_TRANSPORT_OPTIONS)
app.conf.CELERY_ACCEPT_CONTENT = ['json']
app.conf.CELERY_TASK_SERIALIZER = 'json'
app.conf.CELERYD_PREFETCH_MULTIPLIER = 1
app.log.already_setup=True
app.conf.CELERY_ENABLE_REMOTE_CONTROL = False
app.conf.CELERY_DEFAULT_QUEUE = queue_name
#app.task
def convert_file(file_conversion_json):
file_conversion_json_copy = file_conversion_json.copy()
So basically I want to take a value from the dictionary and add it to my log. I have done this successfully in my actual application by using the following code:
import uuid
import logging
import flask
# Generate a new request ID, optionally including an original request ID
def generate_request_id(original_id):
new_id = uuid.uuid4()
if not original_id:
return new_id
else:
new_id = original_id
return new_id
# Returns the current request ID or a new one if there is none
# In order of preference:
# * If we've already created a request ID and stored it in the flask.g context local, use that
# * If a client has passed in the X-Request-Id header, create a new ID with that prepended
# * Otherwise, generate a request ID and store it in flask.g.request_id
def request_id():
if getattr(flask.g, 'request_id', None):
return flask.g.request_id
headers = flask.request.headers
original_request_id = headers.get("X-Request-Id")
new_uuid = generate_request_id(original_request_id)
flask.g.request_id = new_uuid
return new_uuid
class RequestIdFilter(logging.Filter):
# This is a logging filter that makes the request ID available for use in
# the logging format. Note that we're checking if we're in a request
# context, as we may want to log things before Flask is fully loaded.
def filter(self, record):
record.request_id = request_id() if flask.has_request_context() else ''
return True
............
# log.py
import logging.config
import os
LOGGER_CONFIGURATION = {
'version': 1,
'filters': {
'request_id': {
'()': 'config.utils.RequestIdFilter',
},
},
'formatters': {
'standard': {
'format': '%(asctime)s - %(name)s.%(module)s.%(funcName)s:%(lineno)d - %(levelname)s - %(request_id)s - %(message)s',
},
},
'handlers': {
'console': {
'class': 'logging.StreamHandler',
'level': 'INFO',
'filters': ['request_id'],
'formatter': 'standard'
}
},
'loggers': {
'': {
'handlers': ['console'],
'level':'INFO',
},
'app': {
'handlers': ['console'],
'level':'INFO',
},
}
}
logging.config.dictConfig(LOGGER_CONFIGURATION)
But it doesn't work for celery and I have no idea how to add a variable on the fly like that. Any advice?

So I figured out that if I add this to my task:
logFormatter = logging.Formatter('%(asctime)s - %(name)s.%(module)s.%(funcName)s:%(lineno)d - %(levelname)s - Request ID:'+ file_conversion_json['x_request_id'] + ' - %(message)s')
rootLogger = logging.getLogger()
consoleHandler = logging.StreamHandler()
consoleHandler.setFormatter(logFormatter)
rootLogger.addHandler(consoleHandler)
It will add what I want to the log output, but now it duplicated my log messages, I am getting each message twice in the log output....

Writing custom log handler in python

I am trying to extend RotatingFile Handler to say FooBar.
class FooBar(RotatingFileHandler) :
def __init__(self, filename, mode='a', maxBytes=0,backupCount=0, encoding=None, delay=0) :
RotatingHandler.__init__(self, filename, mode, maxBytes, backupCount, encoding, delay)
I configure it using
LOGGING = {
'version': 1,
'disable_existing_loggers': True,
'formatters': {
'verbose': {
'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
},
'simple': {
'format': '%(levelname)s %(asctime)s %(message)s'
},
},
'handlers': {
'file':{
'level': 'ERROR',
'class': 'myhandler.FooBar',
'formatter': 'simple',
'filename': '/tmp/cattle.txt',
'mode': 'a',
'maxBytes': 16,
'backupCount' : 100,
},
#-- Remaining part truncated ###
logging.config.dictConfig(LOGGING) ### === ERROR here
When I use it ; I get an error
File "/usr/lib/python2.7/logging/config.py", line 576, in configure
'%r: %s' % (name, e))
ValueError: Unable to configure handler 'file': global name 'RotatingHandler' is not defined

RotatingHandler is not in scope, so you would need something like this to bring it into scope:
from logging.handlers import RotatingFileHandler
However, have a look at this example:
How to log everything into a file using RotatingFileHandler by using logging.conf file?
You may not need to create your own class to accomplish what you want to do.