I would like to use the following log format:
'format': '{"message": "%(message)s", "user": "%(user)s"}'
However, I would like to call it in two different ways:
log.info("hi", extra={"user": "asmith"})
log.info("hi")
The first log statement works because it provides the user argument, but the second one fails with a KeyError.
Is there any way to make a format string argument optional?
Optional format args replaced with None
Given a fixed logging format string, a custom Formatter class can be used to replace missing arguments with None.
import logging
import re
class CustomFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
arg_pattern = re.compile(r'%\((\w+)\)')
arg_names = [x.group(1) for x in arg_pattern.finditer(self._fmt)]
for field in arg_names:
if field not in record.__dict__:
record.__dict__[field] = None
return super().format(record)
logger = logging.getLogger(__name__)
handler = logging.StreamHandler()
formatter = CustomFormatter('{"message": "%(message)s", "user": "%(user)s"}')
handler.setFormatter(formatter)
logger.setLevel(logging.INFO)
logger.addHandler(handler)
logger.info('hi')
logger.info('hi', extra={"user": "asmith"})
Output
{"message": "hi", "user": "None"}
{"message": "hi", "user": "asmith"}
Dynamically add extra args to logging output
A custom Formatter can dynamically update the format string based on the dictionary passed to extra.
import logging
class ExtraFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
default_attrs = logging.LogRecord(None, None, None, None, None, None, None).__dict__.keys()
extras = set(record.__dict__.keys()) - default_attrs
log_items = ['"message": "%(message)s"']
for attr in extras:
log_items.append(f'"{attr}": "%({attr})s"')
format_str = f'{{{", ".join(log_items)}}}'
self._style._fmt = format_str
return super().format(record)
logger = logging.getLogger(__name__)
handler = logging.StreamHandler()
formatter = ExtraFormatter()
handler.setFormatter(formatter)
logger.setLevel(logging.INFO)
logger.addHandler(handler)
logger.info('hi')
logger.info('hi', extra={"user": "asmith", "number": "42"})
Output
{"message": "hi"}
{"message": "hi", "user": "asmith", "number": "42"}
Related
I need to write a test for a function post_log_filter(record) in the file my_logging.py
#my_logging.py
import os
import sys
import traceback
log_format = '[%(levelname)s] %(asctime)s %(context)s %(pathname)s:%(lineno)d: %(message)s'
log_time_format = '%Y-%m-%dT%H:%M:%S%z'
def post_log_filter(record):
# filter out undesirable logs
logs_to_omit = [
{'filename': 'basehttp.py', 'funcName': 'log_message'}, # annoying logging any response to sys.stderr (even if status is 200)
{'filename': 'options.py', 'funcName': 'construct_change_message'}, # logging `Key not found` if a checkbox was unchecked and isn't present in POST data
]
if any([bool(record.__dict__.viewitems() >= r.viewitems()) for r in logs_to_omit]):
return False
return True
The test that I have written is:
test_my_logging.py
from django.utils.unittest import TestCase
from my_logging import *
import collections
import mock
from mock import patch
import logging
LOGGER = logging.getLogger(__name__)
log_format = '[%(levelname)s] %(asctime)s %(context)s %(pathname)s:%(lineno)d: %(message)s'
log_time_format = '%Y-%m-%dT%H:%M:%S%z'
class TestMyLogging(TestCase):
def test_post_log_filter(self):
self.assertEqual(True, post_log_filter(logging.LogRecord(None, None, None, None, msg=None, args=None, exc_info=None, func=None)))
def test_post_log_filter_false(self):
record = logging.Formatter([
{'filename': 'basehttp.py', 'funcName': 'log_message'}, # annoying logging any response to sys.stderr (even if status is 200)
{'filename': 'options.py', 'funcName': 'construct_change_message'}, # logging `Key not found` if a checkbox was unchecked and isn't present in POST data
])
self.assertEqual(False, post_log_filter(record))
I am testing it for two cases.
For True:
No matter what I pass for post_log_filter(), I get true. So the first test passes.
I don't know what I'm doing wrongly when passing record for testing for False, I get True for this as well. So, the test fails.
How would you recommend me pass the record so I get False.
I am not allowed to change my_logging.py.
I have found the solution.
Modifying the record as below now tests for False.
def test_post_log_filter_false(self):
record = logging.makeLogRecord({'filename': 'basehttp.py', 'funcName': 'log_message'})
self.assertEqual(False, post_log_filter(record))
Both test passes.
I try to add a logging feature to my cloud function. And integrate the logging module with google-cloud-logging. Below is the code I deploy on cloud function:
import logging
import platform
import sys
from datetime import datetime
from google.cloud import logging as gcp_logging
from google.cloud.logging.handlers import CloudLoggingHandler
from google.cloud.logging.resource import Resource
class Singleton(type):
_instances = {}
def __call__(cls, *args, **kwargs):
if cls not in cls._instances:
cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
else:
# uncommnet this if you need initialzation everytime
cls._instances[cls].__init__(*args, **kwargs)
return cls._instances[cls]
class LogManager(object, metaclass=Singleton):
#TODO duplicate log issue
gcp_client = None
gcp_handler = None
log_file = ''
log_options = {
'console': {
'log_level': logging.DEBUG
}
}
default_fmt = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(filename)s - %(lineno)s - %(message)s')
def __new__(cls, options=None):
print('new LOG manager')
return object.__new__(cls)
def __init__(self, options=None):
print('init LOG manager')
if options is not None:
self.log_options = options
if 'gcp' in self.log_options:
if platform.node() and platform.node() != 'localhost':
print('use auth_token when executed on local')
self.gcp_client = gcp_logging.Client.from_service_account_json('MY_SECRET.json')
else:
print('get environment default, use it when executed on cloud')
self.gcp_client = gcp_logging.Client()
def get_logger(self, logger_name, fmt=None, log_tags={}):
# there is a manager keeping all loggers for reusing
logger = logging.getLogger(logger_name)
logger.setLevel(logging.DEBUG)
if logger.hasHandlers():
logger.handlers.clear()
if fmt is not None:
self.formatter = fmt
else:
self.formatter = self.default_fmt
if 'console' in self.log_options:
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(self.log_options['console']['log_level'])
ch.setFormatter(self.formatter)
logger.addHandler(ch) # add the handlers to the logger
if 'gcp' in self.log_options:
gcp_handler = CloudLoggingHandler(
client=self.gcp_client,
resource=Resource(
type=self.log_options['gcp']['log_type'],
labels=self.log_options['gcp']['labels']
)
)
gcp_handler.setFormatter(self.formatter)
gcp_handler.setLevel(self.log_options['gcp']['log_level'])
logger.addHandler(gcp_handler)
return logger
def stop_logging(self):
logging.shutdown()
def hello_world(request):
"""Responds to any HTTP request.
Args:
request (flask.Request): HTTP request object.
Returns:
The response text or any set of values that can be turned into a
Response object using
`make_response <http://flask.pocoo.org/docs/1.0/api/#flask.Flask.make_response>`.
"""
mode = request.args.get('mode', default = 'gcp')
if mode=='console':
print('using default logger setup')
log_manager = LogManager() # use this for default setup
else:
print('using gcp logger setup')
log_manager = LogManager(
options={
'gcp': {
'log_level': logging.INFO,
'log_type': 'cloud_function',
'labels': {
'function_name': 'test_logging',
},
},
}
)
logger = log_manager.get_logger(__name__)
logger.debug('debug msg1')
logger.info('info msg1')
logger.warning('warning msg1')
logger.error('error msg1')
logger.critical('critical msg1')
logger = LogManager().get_logger(__name__)
logger.debug('debug msg2')
logger.info('info msg2')
logger.warning('warning msg2')
logger.error('error msg2')
logger.critical('critical msg2')
log_manager.stop_logging()
return 'ok'
With dependency google-cloud-logging==1.11.0 and google-api-core==1.22.2.
Here is the result when I use StreamHandler:
As you can see, there are 2 types of logging messages:
formatted log output from logger
pure log message
And both of these messages are automatically tagged with a random ID by GCP.
Here is another result when I use CloudLoggingHandler:
As you can see, there are 2 types of logging messages:
formatted log output from logger without random ID tagged by GCP
pure log message tagged with a random ID by GCP
My questions are:
How to resolve the duplicates issue? (either CloudLoggingHandler or StreamHandler)
How is the auto-tagging mechanism work? I am trying to keep these tags since they could be useful for searching (e.g. test_logging & r2b4rhft6z1i in the screenshot)
The last one is a little bit beyond this topic. Should I use singleton in this case? Am I use it right? Is there any improvement for this code snippet?
Sorry for asking so many questions at once. Any suggestion will be appreciated!
I have read Flask - How to store logs and add additional information and so on.
But I don't want to write code like extra={} everywhere.
I try custom logger of FlaskApp by use AppFormatter, but it dosen't work. Here is the code sample:
import logging
from flask import session, Flask
from logging.handlers import RotatingFileHandler
class AppFormatter(logging.Formatter):
def format(self, record):
# fixme: AppFormatter.format is not called
s = super(AppFormatter, self).format(record)
user_id = session.get('user_id', '?')
username = session.get('fullanme', '??')
msg = '{} - {} - {}'.format(s, user_id, username)
return msg
LOG_FORMAT = '[%(asctime)s]%(module)s - %(funcName)s - %(message)s'
defaultFormat = AppFormatter(LOG_FORMAT)
def initLogger(logger, **kwargs):
file = kwargs.pop('file', 'debug.log')
fmt = kwargs.pop('format', defaultFormat)
level = kwargs.pop('level', logging.DEBUG)
maxBytes = kwargs.pop('maxBytes', 10 * 1024 * 1024)
backupCount = kwargs.pop('backupCount', 5)
hdl_file = RotatingFileHandler(file, maxBytes=maxBytes, backupCount=backupCount)
hdl_file.setLevel(level)
logger.addHandler(hdl_file)
for hdl in logger.handlers:
hdl.setFormatter(fmt)
app = Flask(__name__)
initLogger(app.logger)
app.run()
Why AppFormatter.format is not called while app.logger stdout the messages ?
Try this out
class AppFormatter(logging.Formatter):
def format(self, record):
user_id = session.get('user_id', '?')
username = session.get('fullanme', '??')
record.msg = '{} - {} - {}'.format(record.getMessage(), user_id, username)
return super(AppFormatter, self).format(record)
I am using the standard Python logging package to write logfiles in a Flask application. I would like to add the user ip address and if the user is authenticated I would also like to log the user id. Does anybody know how to do this?
Right now my formatter looks like this
fmt = ('%(asctime)s - %(name)s - %(levelname)s - '
'%(filename)s:%(lineno)s - %(funcName)20s() - %(message)s')
formatter = logging.Formatter(fmt)
Ok I manager to do it by adding a costum filter
class ContextualFilter(logging.Filter):
def filter(self, log_record):
''' Provide some extra variables to give our logs some better info '''
log_record.utcnow = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S,%f %Z')
log_record.url = request.path
log_record.method = request.method
# Try to get the IP address of the user through reverse proxy
log_record.ip = request.environ.get('HTTP_X_REAL_IP', request.remote_addr)
if current_user.is_anonymous():
log_record.user_id = 'guest'
else:
log_record.user_id = current_user.get_id()
return True
which can be added to the logger as
std_logger.addFilter(ContextualFilter())
the formatter than should be something like
fmt = ('%(utcnow)s - %(levelname)s - '
'user_id:%(user_id)s - ip:%(ip)s - %(method)s - %(url)s - '
'%(filename)s:%(lineno)s - %(funcName)20s() - %(message)s')
I am developing a Amazon Lex Chatbot in AWS Lambda in python which will make a API post call and get a response in JSON string as below
'{"_id":"598045d12e1f98980a00001e","unique_id":"ed7e4e17c7db499caee576a7761512","cerebro":{"_id":"59451b239db9fa8b0a000004","acc_id":"533a9f0d2eda783019000002","name":"cerebro","access_id":"g01n0XTwoYfEWSIP","access_token":"3Yxw8ZiUlfSPsbEVLI6Z93vZyKyBFFIV"},"bot":{"_id":"59452f42dbd13ad867000001","name":"helloword"},"rundata":{"arguments":"","target":"local"},"state":"created","queue_id":null,"s_ts":null,"e_ts":null,"response":{},"responses":[],"summary":null,"resolve_incident":false,"err":null}'
But i am interested in the id value only so i am converting the json into a dictionary as below and getting the id value
res = requests.post(botrun_api, json=botrun_payload, headers=headers)
data = json.loads(res.content)
new_id=json_data.get('_id', None)
return new_id
If i am testing the code in Lambda console i am getting the output
Output in AWS Lambda console
But i am getting output as below in my Chatbot
I was unable to process your message. DependencyFailedException: Invalid Lambda Response: Received invalid response from Lambda: Can not construct instance of IntentResponse: no String-argument constructor/factory method to deserialize from String value ('59832ba22e1f98980a00009b') at [Source: "59832ba22e1f98980a00009b"; line: 1, column: 1]
My Source code as below:
import math
import dateutil.parser
import datetime
import time
import os
import logging
import requests
import uuid
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
""" --- Helpers to build responses which match the structure of the necessary dialog actions --- """
def get_slots(intent_request):
return intent_request['currentIntent']['slots']
def elicit_slot(session_attributes, intent_name, slots, slot_to_elicit, message):
return {
'sessionAttributes': session_attributes,
'dialogAction': {
'type': 'ElicitSlot',
'intentName': intent_name,
'slots': slots,
'slotToElicit': slot_to_elicit,
'message': message
}
}
def close(session_attributes, fulfillment_state, message):
response = {
'sessionAttributes': session_attributes,
'dialogAction': {
'type': 'Close',
'fulfillmentState': fulfillment_state,
'message': message
}
}
return response
def delegate(session_attributes, slots):
return {
'sessionAttributes': session_attributes,
'dialogAction': {
'type': 'Delegate',
'slots': slots
}
}
""" --- Helper Functions --- """
def parse_int(n):
try:
return int(n)
except ValueError:
return float('nan')
def build_validation_result(is_valid, violated_slot, message_content):
if message_content is None:
return {
"isValid": is_valid,
"violatedSlot": violated_slot,
}
return {
'isValid': is_valid,
'violatedSlot': violated_slot,
'message': {'contentType': 'PlainText', 'content': message_content}
}
def APIbot(intent_request):
"""
Performs dialog management and fulfillment for cluster configuration input arguments.
Beyond fulfillment, the implementation of this intent demonstrates the use of the elicitSlot dialog action
in slot validation and re-prompting.
"""
value1 = get_slots(intent_request)["myval1"]
value2 = get_slots(intent_request)["myval2"]
intense_type = get_slots(intent_request)["Instance"]
source = intent_request['invocationSource']
api_endpoint = 'url'
api_creds = {
'apiuser': 'user',
'apikey': 'key'
}
#trigger a bot run
botrun_api = api_endpoint + '/botruns'
botrun_payload = {
"botname":"helloword",
"arguments":"",
"target":"local",
"unique_id": uuid.uuid4().hex[:30] #unique run id - 30 chars max
}
headers = {
'Content-Type': 'application/json',
'Authorization': 'Key apiuser=%(apiuser)s apikey=%(apikey)s' % api_creds
}
res = requests.post(botrun_api, json=botrun_payload, headers=headers)
data = json.loads(res.content)
new_id=json_data.get('_id', None)
return new_id
# Instiate a cluster setup, and rely on the goodbye message of the bot to define the message to the end user.
# In a real bot, this would likely involve a call to a backend service.
return close(intent_request['sessionAttributes'],
'Fulfilled',
{'contentType': 'PlainText',
'content': 'Thanks, your values are {} and {} '.format(value1, value2)})
""" --- Intents --- """
def dispatch(intent_request):
"""
Called when the user specifies an intent for this bot.
"""
logger.debug('dispatch userId={}, intentName={}'.format(intent_request['userId'], intent_request['currentIntent']['name']))
intent_name = intent_request['currentIntent']['name']
# Dispatch to your bot's intent handlers
if intent_name == 'my_Values':
return APIbot(intent_request)
raise Exception('Intent with name ' + intent_name + ' not supported')
""" --- Main handler --- """
def lambda_handler(event, context):
"""
Route the incoming request based on intent.
The JSON body of the request is provided in the event slot.
"""
# By default, treat the user request as coming from the America/New_York time zone.
os.environ['TZ'] = 'America/New_York'
time.tzset()
logger.debug('event.bot.name={}'.format(event['bot']['name']))
return dispatch(event)
Please help me in resolving this thanks in advance :)
Lambda is showing your function succeeding when only the new_id is returned because it does not care about the format of the response.
When connected to AWS Lex, the response must be in the AWS defined response format.
In your above example, you can pass the new_id through in the close method, to output the response via Lex:
return close(intent_request['sessionAttributes'],
'Fulfilled',
{'contentType': 'PlainText',
'content': str(new_id)}
You'll also need to remove the return new_id statement.