We Keep Coding

How add my custom handler to python logging.config.dictConfig()

I try to add my handler to send errors to telegram.
I create my own handler:
#myLogHandlers.py
import logging
from config import config
from pyrogram import Client
from time import time, sleep
logger = logging.getLogger(__name__)
class TelegramHandler(logging.Handler):
"""Loggig handler for sending logs to telegram"""
def __init__(self, user_id_list: list[int],
chat_id_list: list[int]=[-525253371],
API_ID=config.API_ID,
API_HASH=config.API_HASH,
BOT_TOKEN=config.BOT_TOKEN,
timeout: int = 10):
super().__init__()
self.chat_id_list = chat_id_list
self.API_ID = API_ID
self.API_HASH = API_HASH
self.BOT_TOKEN = BOT_TOKEN
self.timeout = timeout
self.appBot = self._get_app()
def _get_app(self):
return Client('appBotHandler', api_id=self.API_ID, api_hash=self.API_HASH, bot_token=self.BOT_TOKEN)
def emit(self, record):
msg = self.format(record)
for chat_id in self.chat_id_list:
t0 = time()
while time() - t0 < self.timeout:
try:
self.appBot.send_message(chat_id, msg)
break
except Exception as ex:
logger.exception("Exception while sending %s to %s:", msg, chat_id)
sleep(1)
Then I config logging in main code module:
#serverBot.py
import logging.config
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'default': {
'format': '{asctime} {levelname} {module}:{funcName}:{lineno} '
'{message}',
'style': '{'
'handlers': {
'console': {
'level': 'DEBUG',
'class': 'logging.StreamHandler',
'formatter': 'default'
},
'telegram': {
(): 'myLogHandlers.TelegramHandler',
'level': 'ERROR',
'formatter': 'default'
}
},
'loggers': {
'serverBot': {
'handlers': ['telegram', 'console'],
'level': 'DEBUG',
'propagate': True
}
}
}
logging.config.dictConfig(LOGGING)
logger = logging.getLogger('serverBot')
So I get error:
Traceback (most recent call last):
File "/usr/local/Cellar/python#3.9/3.9.5/Frameworks/Python.framework/Versions/3.9/lib/python3.9/logging/config.py", line 564, in configure
handler = self.configure_handler(handlers[name])
File "/usr/local/Cellar/python#3.9/3.9.5/Frameworks/Python.framework/Versions/3.9/lib/python3.9/logging/config.py", line 722, in configure_handler
klass = self.resolve(cname)
File "/usr/local/Cellar/python#3.9/3.9.5/Frameworks/Python.framework/Versions/3.9/lib/python3.9/logging/config.py", line 382, in resolve
name = s.split('.')
AttributeError: 'NoneType' object has no attribute 'split'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/.../serverBot.py", line 10, in <module>
logging.config.dictConfig(config.LOGGING)
File "/usr/local/Cellar/python#3.9/3.9.5/Frameworks/Python.framework/Versions/3.9/lib/python3.9/logging/config.py", line 809, in dictConfig
dictConfigClass(config).configure()
File "/usr/local/Cellar/python#3.9/3.9.5/Frameworks/Python.framework/Versions/3.9/lib/python3.9/logging/config.py", line 571, in configure
raise ValueError('Unable to configure handler '
ValueError: Unable to configure handler 'telegram'
Process finished with exit code 1
How can I say to logging.config where my handler is?

You need to follow the documentation more closely. For example, the key for a custom handler is documented as '()' (string), not () (empty tuple). Also, where are you providing the user_id_list that the handler needs in order to be instantiated? Without that, the instantiation would fail, even if it got that far.

Python logging with {} format using logging.config.dictConfig

I'm trying to use the string.format() style with python logging module. I copied some examples from this site and modified it with the the new formatter. I'd like to use logging.config.dictConfig to specify the logging format.
My script is:
import sys, logging, logging.config
DEFAULT_LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'standard': {
'format': '{message}',
'style': '{' # use string.format()
},
},
'handlers': {
'default': {
'level': 'INFO',
'formatter': 'standard',
'class': 'logging.StreamHandler',
'stream': 'ext://sys.stdout', # Default is stderr
},
},
'loggers': {
'__main__': { # if __name__ == '__main__'
'handlers': ['default'],
'level': 'DEBUG',
'formatter': 'standard'
},
'': {
'handlers': ['default'],
'level': 'INFO',
'formatter': 'standard'
},
}
}
logger = logging.getLogger(__name__)
logging.config.dictConfig(DEFAULT_LOGGING)
if __name__ == '__main__':
logger.info('Hello, {}', 'log')
sys.exit(0)
But an exception raised:
--- Logging error --- Traceback (most recent call last): File "/usr/lib/python3.8/logging/init.py", line 1081, in emit
msg = self.format(record) File "/usr/lib/python3.8/logging/init.py", line 925, in format
return fmt.format(record) File "/usr/lib/python3.8/logging/init.py", line 664, in format
record.message = record.getMessage() File "/usr/lib/python3.8/logging/init.py", line 369, in getMessage
msg = msg % self.args TypeError: not all arguments converted during string formatting Call stack: File "test/dbg/log.py", line
48, in
logger.info('Hello, {}', 'log') Message: 'Hello, {}' Arguments: ('log',)
For some reason python is trying to format using % formatter (msg = msg % self.args). How can I modify my script to point to the correct formatting? Any help will be appreciated.
Thanks in advance.
-Uri

One way of addressing this issue is to create a thin adapter around the built-in logging formatter(s).
Example:
import sys, logging, logging.config
class standardFormatterFactory(logging.Formatter):
def __init__(self, fmt, datefmt, style, validate=False):
try:
# 3.x
super(standardFormatterFactory, self).__init__(fmt=fmt, datefmt=datefmt, style=style, validate=validate)
except:
# 2.7
super(standardFormatterFactory, self).__init__(fmt=fmt, datefmt=datefmt)
self._usesTime = self._fmt.find('%(asctime)') >= 0 or self._fmt.find('{asctime}') >= 0
def usesTime(self):
'''Override logging.Formatter.usesTime()'''
return self._usesTime
def format(self, record):
'''Override logging.Formatter.format()'''
try:
return super(standardFormatterFactory, self).format(record=record)
except TypeError as ex:
# message contains {} formate specifiers
record.message = record.msg.format(*record.args)
if self.usesTime():
record.asctime = self.formatTime(record, self.datefmt)
return self._fmt.format(**record.__dict__)
DEFAULT_LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'propagate': False,
'formatters': {
'standard': {
'()' : standardFormatterFactory,
'fmt': '{asctime}-{levelname}: {message}',
'datefmt': '%Y-%m-%d %H:%M:%S',
'style': '{'
},
},
'handlers': {
'default': {
'level': 'DEBUG',
'formatter': 'standard',
'class': 'logging.StreamHandler',
'stream': 'ext://sys.stdout', # Default is stderr
},
},
'loggers': {
'__main__': { # if __name__ == '__main__'
'handlers': ['default'],
'level': 'DEBUG',
'formatter': 'standard'
},
}
}
logger = logging.getLogger(__name__)
logging.config.dictConfig(DEFAULT_LOGGING)
if __name__ == '__main__':
logger.info('Hello, {}', 'log')
sys.exit(0)

django database query log linenumber

I am logging my database queries in Django along with the pathname and linenumber.
Right now i am getting these logs:
07/Dec/2018 14:25:00 DEBUG django.db.backends utils **/Users/XXXXX/.idea/lib/python2.7/site-packages/django/db/backends/utils.py:89**
(0.340) SELECT "metadata"."metaname", "metadata"."description", "metadata"."attributes" FROM "metadata" WHERE "metadata"."metaname" = 'date_type'; args=('date_type',)
For all queries, I am getting the same path and line number. Is there any way I can capture the line number from my main application instead of the one from utils.
Current logging Implementation:
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'handlers': {
'console': {
'level': 'DEBUG',
'class': 'logging.StreamHandler',
'formatter': 'color'
},
},
'loggers': {
'django.db.backends': {
'handlers': ['console'],
'level': 'DEBUG',
'propogate': True,
}
}
}
Using python 2.7 and django 1.9

The slightly optimized version from User #will-keeling
Logging configuration for Django for output line numbers for each db request.
Note: If you want to use it for tests you need to set DEBUG=True for tests How do you set DEBUG to True when running a Django test?
import logging
import traceback
from django.conf import settings
class StackInfoHandler(logging.StreamHandler):
trim = 5
def emit(self, record):
super(StackInfoHandler, self).emit(record)
trace = traceback.format_stack()
stack1 = [str(row) for row in trace]
stack2 = [s for s in stack1 if settings.BASE_DIR in s and 'format_stack' not in s]
stack3 = [s for s in stack2 if 'test' not in s]
if not stack3:
stack3 = stack2 # include test call
if stack3:
stack4 = ''.join(stack3[-self.trim:]) # take only last records
stack5 = f"Stack {self.terminator} {''.join(stack4)}"
self.stream.write(stack5)
self.stream.write(self.terminator)
self.flush()
Logging Config (partitial)
LOGGING = {
'handlers': {
'db-console': {
'level': 'DEBUG',
'class': 'settings.local.StackInfoHandler', # Reference the custom handler
'formatter': 'simple',
},
'loggers': {
'django.db.backends': {
'handlers': ['db-console'],
'level': 'DEBUG',
'propagate': False
},
}
}
}
This will show you only stack trace from your Django codebase like below
[2020-05-25 17:49:17,977]: (0.000) INSERT INTO `contacts_contactscount` (`user_id`, `date`, `amount`) VALUES (338, '2020-05-25 17:49:17', 7); args=[338, '2020-05-25 17:49:17', 7]
Stack
File "<project-root>/api/views/contacts.py", line 164, in create
Contact.objects.filter(pk__in=to_delete).delete()
File "<project-root>/<folder>/contacts/models.py", line 54, in delete
create_deletion_log.delay(obj, deleted_timestamp)
File "<project-root>/<folder>/contacts/tasks.py", line 31, in create_deletion_log
contact.save()
File "<project-root>/<folder>/contacts/models.py", line 118, in save
Contact.objects.contacts_added_hook(self.user)
File "<project-root>/<folder>/contacts/models.py", line 67, in contacts_added_hook
current_total = user.profile.contacts_total
File "<project-root>/<folder>/profile/models.py", line 631, in contacts_total
ContactsCount.objects.create(user=self.user, amount=count)

I'm guessing that you're trying to determine which lines in your application are responsible for running which queries.
One way to achieve this would be to create a custom handler that prints out the current stack at the point where Django logs the query. That would allow you to see which line in your application is executing.
You could create a custom handler such as:
import logging
import traceback
class StackInfoHandler(logging.StreamHandler):
trim = 5
def emit(self, record):
super(StackInfoHandler, self).emit(record)
stack = ''.join(
str(row) for row in traceback.format_stack()[:-self.trim]
)
self.stream.write(stack)
And then in your logging config, you could just switch the handler class to use the StackInfoHandler:
'handlers': {
'console': {
'level': 'DEBUG',
'class': 'my.package.StackInfoHandler', # Reference the custom handler
'formatter': 'color'
},
},
Note that the StackInfoHandler trims 5 lines off the stack so that it doesn't show you stack frames from the logging framework itself. You might need to tweak this number (5 works for me locally).

How to set logger class universally in Django App

I have a logger that just adds log_line if none is provided when calling logger.debug
class UrlLogger(logging.Logger):
def _log(self, level, msg, args, exc_info=None, extra=None):
if extra is None:
extra = {'log_line':' '}
super(UrlLogger, self)._log(level, msg, args, exc_info, extra)
I have about 20 different modules that I need to go into and add
logging.setLoggerClass(UrlLogger)
How can I set this as default in the settings?
My settings currently look like this:
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'verbose': {
'format': '%(levelname)s %(name)s %(asctime)s %(module)s %(process)d %(thread)d %(pathname)s#%(lineno)s: %(message)s'
},
'simple': {
'format': '[%(levelname)8s] [%(asctime)s] %(module)10s/%(filename)s:%(log_line)s - %(message)s',
'datefmt': '%d-%m-%Y %H:%M:%S'
},
},
'handlers': {
'console': {
'class': 'logging.StreamHandler',
'level': LOG_LEVEL,
'formatter': 'simple'
},
},
'loggers': {
'': {
'level': LOG_LEVEL,
'handlers': ['console'],
'class': ['UrlLogger'],
},
As you can see, I tried setting it here, but I don't think that's working because I am still getting errors.
Message: 'Not Found: %s'
Arguments: ('/tool_job_list/1/1fa119372de6/job/check/jexecution/report',)
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.6/logging/__init__.py", line 992, in emit
msg = self.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 838, in format
return fmt.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 578, in format
s = self.formatMessage(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 547, in formatMessage
return self._style.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 391, in format
return self._fmt % record.__dict__
KeyError: 'log_line'
This error occurs when the front end makes failed API calls (wrong uri) to the django backend. I'm thinking that the failed api call is logging the error without using my UrlLogger (otherwise there would not be a key error unless there is an error in my Logger Class).
What am I doing wrong here?

JIRA not recognizing dictionary object in cURL call

This is the error I am currently getting while using the jira-python python module to automate some logging to JIRA.
Traceback (most recent call last):
File "/home/csd-user/test/libs/utils/butler.py", line 217, in <module>
main()
File "/home/csd-user/test/libs/utils/butler.py", line 214, in main
b.log_bug_exec(url)
File "/home/csd-user/test/libs/utils/butler.py", line 47, in log_bug_exec
cls.process_file(stdout)
File "/home/csd-user/test/libs/utils/butler.py", line 108, in process_file
cls.submit_bug(bug)
File "/home/csd-user/test/libs/utils/butler.py", line 207, in submit_bug
iss = cls.my_server.create_issue(fields=bug.json_dict['fields'])
File "/opt/clearsky/lib/python2.7/site-packages/jira/client.py", line 706, in create_issue
r = self._session.post(url, data=json.dumps(data))
File "/opt/clearsky/lib/python2.7/site-packages/jira/resilientsession.py", line 81, in post
return self.__verb('POST', url, **kwargs)
File "/opt/clearsky/lib/python2.7/site-packages/jira/resilientsession.py", line 74, in __verb
raise_on_error(r, verb=verb, **kwargs)
File "/opt/clearsky/lib/python2.7/site-packages/jira/utils.py", line 120, in raise_on_error
r.status_code, error, r.url, request=request, response=r, **kwargs)
# This is the important part...
jira.utils.JIRAError: JiraError HTTP 400
text: data was not an object
url: https://jira.clearsky-data.net/rest/api/2/issue
My problem is that, as far as I can see, the dict object that I am passing it is perfectly valid.
BUG FIELDS :: {'environment': 'node => 62-qa-driver12 (M3 - HA Only)\nversion => \nurl => https://jenkins.clearsky-data.net/job/BugLoggerTest/144/\ntimestamp => 2015-06-29_11-11-15\njob name => BugLoggerTest\nbuild number => 144\nversion number => Not present. Check git hash. Maybe add in processing of full failure list!\n', 'description': '', 'summary': 'Fill in Something', 'project': {'key': 'QABL'}, 'assignee': 'qa-auto', 'issuetype': {'name': 'Bug'}, 'priority': {'name': 'Major'}}
CLASS :: <type 'dict'>
Is formatted by this...
# creating JSON object (bug should not have to be changed after initialization)
self.json_dict = ( {"fields": {
"project": {'key': self.project},
"issuetype": {'name': self.issue_type},
"priority": {'name': self.priority},
"assignee": self.assignee,
"environment": self.environment,
"description": self.description,
"summary": self.summary } } )
This is the call to create the issue where the error is being thrown:
iss = cls.my_server.create_issue(fields=bug.json_dict['fields'])
# This calls a cURL POST or PUT command.

Your assignee is not what jira expects:
"assignee": self.assignee,
should read
"assignee": {'name': self.assignee}
I saw it in the update example
issue.update(summary='new summary', description='A new summary was added')
issue.update(assignee={'name': 'new_user'})