Creating a custom logger for my purpose using python which can be used across different modules just by importing and calling a custom_log method.
This is MyLogger.py script.
import datetime
import logging
import logging.handlers
import os
import colorlog
from pathlib import Path
class MyLogger(logging.Logger):
def __init__(self, verbose=1):
log_dir_path = Path("../logs")
file_name_format = '{year:04d}{month:02d}{day:02d}-{hour:02d}{minute:02d}{second:02d}.log'
file_msg_format = '%(asctime)s %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s'
console_msg_format = '%(asctime)s %(levelname)-8s: %(message)s'
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
if (verbose == 1):
max_bytes = 1024 ** 2
backup_count = 100
t = datetime.datetime.now()
file_name = file_name_format.format(year=t.year, month=t.month, day=t.day, hour=t.hour, minute=t.minute,
second=t.second)
file_name = os.path.join(log_dir_path, file_name)
file_handler = logging.handlers.RotatingFileHandler(filename=file_name, maxBytes=max_bytes, backupCount=backup_count)
file_handler.setLevel(logging.DEBUG)
file_formatter = logging.Formatter(file_msg_format)
file_handler.setFormatter(file_formatter)
logger.addHandler(file_handler)
if (verbose == 1):
cformat = '%(log_color)s' + console_msg_format
colors = {'DEBUG': 'green', 'INFO': 'cyan', 'WARNING': 'bold_yellow', 'ERROR': 'bold_red',
'CRITICAL': 'bold_purple'}
date_format = '%Y-%m-%d %H:%M:%S'
formatter = colorlog.ColoredFormatter(cformat, date_format, log_colors=colors)
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.DEBUG)
stream_handler.setFormatter(formatter)
logger.addHandler(stream_handler)
def custom_log(self, level, msg):
logging.log(getattr(logging,level),msg)
I have 2 other scripts within the same directory as below, just need to initialize MyLogger() in Test1.py at beginning of the test and expecting to use custom_log in all other scripts. Missing something here to make this work,either in the way of initializing or the way of importing. Any support how to get this done.
Test1.py
from MyLogger import MyLogger
class StartTest():
def __init__(self):
MyLogger.custom_log('DEBUG','Debug messages are only sent to the logfile.')
if __name__ == '__main__':
MyLogger()
StartTest()
Test2.py
from MyLogger import MyLogger
class Test():
def __init__(self):
MyLogger.custom_log('DEBUG','Debug messages are only sent to the logfile.')
def TestMethod(self):
MyLogger.custom_log('INFO','Debug messages are only sent to the logfile.')
You can achieve it by creating a logger with a specific name instead of using root logger. logging.log() uses always the root logger. So, you can define a logger with a specific name instead of creating a new Logging channel.
An example which is inline with your need
class MyLogger:
logger: logging.Logger = None
#staticmethod
def configure(verbose=1):
log_dir_path = Path("../logs")
file_name_format = '{year:04d}{month:02d}{day:02d}-{hour:02d}{minute:02d}{second:02d}.log'
file_msg_format = '%(asctime)s %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s'
console_msg_format = '%(asctime)s %(levelname)-8s: %(message)s'
logger = logging.getLogger("mylogger")
logger.setLevel(logging.DEBUG)
cformat = '%(log_color)s' + console_msg_format
colors = {'DEBUG': 'green', 'INFO': 'cyan', 'WARNING': 'bold_yellow', 'ERROR': 'bold_red',
'CRITICAL': 'bold_purple'}
date_format = '%Y-%m-%d %H:%M:%S'
formatter = colorlog.ColoredFormatter(cformat, date_format, log_colors=colors)
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.DEBUG)
stream_handler.setFormatter(formatter)
logger.addHandler(stream_handler)
MyLogger.logger = logger
if __name__ == '__main__':
MyLogger.configure()
MyLogger.logger.error("debug message")
Once the MyLogger.configure is called, then you can use the MyLogger.logger.* any where in your app / scripts.
This can be done without a helper class as well. Create a logger configuration file. Configure your custom logger with a different name rather than root, and from your code always call logging.getLogger("name-of-logger") to create a logging instance.
Related
My main program logs to its own log file and the sub-process should have its own log file.
I replaced the logger object inside the multiprocessing process, but the logging data from the sub-process is additionally redirected to the main log file.
How can I prevent this?
The structure looks like this:
import logging
import sys
import os
from pathlib import Path
import multiprocessing
import time
import requests
class ProcessFilter(logging.Filter):
"""Only accept log records from a specific pid."""
def __init__(self, pid):
self._pid = pid
def filter(self, record):
return record.process == self._pid
def create_logger(file):
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
log.addFilter(ProcessFilter(pid=os.getpid()))
file_handler = logging.FileHandler(file)
stream_handler = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter('[%(asctime)s] %(levelname)s [%(filename)s.%(funcName)s:%(lineno)d] %(message)s',
datefmt='%a, %d %b %Y %H:%M:%S')
file_handler.setFormatter(formatter)
stream_handler.setFormatter(formatter)
log.addHandler(file_handler)
log.addHandler(stream_handler)
return log
def subprocess_init():
global log
sub_log_file = str(Path.home()) + '/logfile_sub.log'
log = create_logger(sub_log_file)
do_subprocess_stuff()
def do_subprocess_stuff():
count = 0
while True:
create_log("subprocess", count)
time.sleep(5)
count += 1
def main_tasks():
num = 10
while num > 0:
create_log("main", num)
time.sleep(5)
num -= 1
def create_log(text, num):
log.debug(text + " log %s", num)
if __name__ == '__main__':
file = str(Path.home()) + '/logfile.log'
log = create_logger(file)
sub_process = multiprocessing.Process(target=subprocess_init, args=())
sub_process.daemon = True
sub_process.start()
main_tasks()
I am simply translating this answer to fit multiprocessing.
import logging
class ProcessFilter(logging.Filter):
"""Only accept log records from a specific pid."""
def __init__(self, pid):
self._pid = pid
def filter(self, record):
return record.process == self._pid
import logging
import os
def create_logger(file):
log = logging.getLogger('') # why use this logger and not __name__ ?
log.setLevel(logging.DEBUG)
log.addFilter(ProcessFilter(pid=os.getpid())) # logger wide filter
file_handler = logging.FileHandler(file)
stream_handler = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter('[%(asctime)s] %(levelname)s [%(filename)s.%(funcName)s:%(lineno)d] %(message)s',
datefmt='%a, %d %b %Y %H:%M:%S')
file_handler.setFormatter(formatter)
stream_handler.setFormatter(formatter)
log.addHandler(file_handler)
log.addHandler(stream_handler)
return log
NB. you can also put the filter on a specific handler
I have one "basic" logging configuration that logs to stdout in my project.
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.INFO)
SERVER_FORMAT = "[%(asctime)s] %(levelname)s [%(name)s:%(filename)s:%(lineno)d] %(message)s"
DATETIME_FORMAT = "%d/%b/%Y %H:%M:%S"
logging.basicConfig(
level=logging.INFO,
format=SERVER_FORMAT,
datefmt=DATETIME_FORMAT,
handlers=[stream_handler])
I have also another logger user_logger that should not print anything. Instead, it should store log entries in a variable.
user_logger = logging.getLogger('user_logger')
user_logger.setLevel(logging.INFO)
log_capture_string = io.StringIO()
log_capture_string_handler = logging.StreamHandler(log_capture_string)
log_capture_string_handler.setLevel(logging.INFO)
log_capture_string_handler.setFormatter(logging.Formatter(USER_FORMAT))
user_logger.handlers = [log_capture_string_handler]
The problem is that when I call:
user_logger.info('This should only be in "log_capture_string"')
it prints it to the console.
Do you know how to avoid that?
You should set user_logger.propagate = False. logging.propagate docs
If this evaluates to false, logging messages are not passed to the handlers of ancestor loggers.
So, your root logger will not send any data to stderr.
This example outputs nothing
import io
import logging
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.INFO)
SERVER_FORMAT = "[%(asctime)s] %(levelname)s [%(name)s:%(filename)s:%(lineno)d] %(message)s"
DATETIME_FORMAT = "%d/%b/%Y %H:%M:%S"
logging.basicConfig(
level=logging.INFO,
format=SERVER_FORMAT,
datefmt=DATETIME_FORMAT,
handlers=[stream_handler])
user_logger = logging.getLogger('user_logger')
user_logger.propagate = False
user_logger.setLevel(logging.INFO)
log_capture_string = io.StringIO()
log_capture_string_handler = logging.StreamHandler(log_capture_string)
log_capture_string_handler.setLevel(logging.INFO)
log_capture_string_handler.setFormatter(logging.Formatter(SERVER_FORMAT))
user_logger.handlers = [log_capture_string_handler]
user_logger.info('This should only be in "log_capture_string"')
I have written a below code to write the logs into cloudwatch using watchtower.
import os
import sys
import time
import boto3
import watchtower
import logging.handlers
from scripts.config import app_configurations
def fast_logging():
try:
boto3_session = boto3.session.Session()
LOG_GROUP = "Fast-Logging"
log_level = DEBUG
stream_name = os.path.join("fast_logging"+ "_" + time.strftime("%Y%m%d") + '.log')
logger = logging.getLogger("Fast-Logger")
logger.setLevel(log_level)
formatter = logging.Formatter('%(name)s - %(levelname)s - %(filename)s - %(module)s: %(funcName)s: '
'%(lineno)d - %(message)s')
log_handler = watchtower.CloudWatchLogHandler(log_group=LOG_GROUP, boto3_session=boto3_session,
stream_name=stream_name)
log_handler.setFormatter(formatter)
logger.addHandler(log_handler)
return logger
except Exception as e:
raise e
logger = fast_logging()
The above code is working for the normal python code but not able to dump logs into cloudwatch stream for the logs in Fast API services.
I found that this code works for me
import os
import time
import boto3
import watchtower
import logging.handlers
def fast_logging():
try:
LOG_GROUP = "Fast-Logging"
log_level = "INFO"
stream_name = os.path.join("fast_logging"+ "_" + time.strftime("%Y%m%d") + '.log')
logger = logging.getLogger("Fast-Logger")
logger.setLevel(log_level)
formatter = logging.Formatter('%(name)s - %(levelname)s - %(filename)s - %(module)s: %(funcName)s: '
'%(lineno)d - %(message)s')
log_handler = watchtower.CloudWatchLogHandler(log_group=LOG_GROUP,
stream_name=stream_name)
log_handler.setFormatter(formatter)
logger.addHandler(log_handler)
return logger
except Exception as e:
raise e
logger = fast_logging()
logger.info("test this")
import logging
import sys
class A(object):
def __init__(self):
ch = logging.StreamHandler(stream=sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s %(name)s %(levelname)s: %(message)s")
ch.setFormatter(formatter)
logger = logging.getLogger("logger_a")
logger.setLevel(logging.DEBUG)
logger.addHandler(ch)
self.logger = logger
def Xprint(self):
self.logger.info("this log a!!")
Xprint()
def Xprint():
logger = logging.getLogger("logger_b")
print logger.info("this log b!!")
a = A()
a.Xprint()
the output:
2019-10-17 19:02:20,574 logger_a INFO: this log a!!
None
why doesn't logger_b print anything?
The default loglevel is WARNING. If you want to make the logger_b to log too, then you need to do something like,
$ cat log.py
import logging
import sys
class A(object):
def __init__(self):
ch = logging.StreamHandler(stream=sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s %(name)s %(levelname)s: %(message)s")
ch.setFormatter(formatter)
logger = logging.getLogger("logger_a")
logger.setLevel(logging.DEBUG)
logger.addHandler(ch)
self.logger = logger
self.handler = ch
def Xprint(self):
self.logger.info("this log a!!")
Xprint(self.handler)
def Xprint(handler):
logger = logging.getLogger('logger_b') # no handler is configured yet
logger.setLevel(logging.DEBUG) # set the level
logger.addHandler(handler) # added handler
logger.info("this log b!!")
a = A()
a.Xprint()
Output:
$ python log.py
2019-10-17 17:02:26,418 logger_a INFO: this log a!!
2019-10-17 17:02:26,418 logger_b INFO: this log b!!
I am trying to log the actions in a function and I have written the following function to log responses to different files based on the type of response i.e. info,error,debug and warning.
logging.basicConfig(filename='indivi_service.log',
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' )
def setup_logger(logger_name, log_file, level=logging.DEBUG):
l = logging.getLogger(logger_name)
formatter = logging.Formatter()
fileHandler = logging.FileHandler(log_file)
fileHandler.setFormatter(formatter)
handler = TimedRotatingFileHandler(logger_name,
when="d",
interval=1,
backupCount=100)
l.setLevel(level)
l.addHandler(fileHandler)
l.addHandler(handler)
setup_logger('debug', r'debug')
setup_logger('error', r'error')
setup_logger('warning', r'warning')
setup_logger('info', r'info')
debug = logging.getLogger('debug')
error = logging.getLogger('error')
warning = logging.getLogger('warning')
info = logging.getLogger('info')
class Info(APIHandler):
#gen.coroutine
def post(self):
req = json.loads(self.request.body)
resp, content = client(item_id=req['item_id'])
debug.debug(content)
info.info(hello world)
warning.warn('warning message')
error.error('error message')
The problem that I am facing is that a response is printed twice each time I call a function.
for example:
info.log
hello world
hello world
Can anyone tell me why is it happening like. This is the case with all the log files.
Thanks
try:
import logging
import logging.handlers
logging.basicConfig(filename='indivi_service.log',
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' )
def setup_logger(logger_name, log_file, level=logging.DEBUG):
l = logging.getLogger(logger_name)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler = logging.handlers.TimedRotatingFileHandler(str(log_file)+'.log', when="d", interval=1, backupCount=100)
handler.setFormatter(formatter)
l.setLevel(level)
l.addHandler(handler)
setup_logger('debug', r'debug')
setup_logger('error', r'error')
setup_logger('warning', r'warning')
setup_logger('info', r'info')
debug = logging.getLogger('debug')
error = logging.getLogger('error')
warning = logging.getLogger('warning')
info = logging.getLogger('info')
if __name__ == "__main__":
info.info('hello world')
error.info('hello world')
after run this script, file info.log has one 'hello world' and error.log also has only one 'hello world'.