I have a problem with the python logging library. I have a flask application that uses the following class (_WrapperLog) to write logs to a file, the library correctly writes logs to today's file and cleans up files older than 7 days correctly. the problem is that when the log rotation occurs, the old files remain saved but their content disappears (as in the example below), only the first 2/4 lines of the file remain, and then nothing, what could be due ?
import logging
from logging.handlers import TimedRotatingFileHandler
import os
from datetime import datetime
from os import name as os_name
from sys import stdout as sys_stdout
class _WrapperLog(object):
class __Log:
logger = None
loggerNameTimeStamp=None
def __init__(self):
self.logger = self.get_logger()
def __str__(self):
return str(self)
def get_logger(self):
if datetime.now().strftime("%Y-%m-%d") == self.loggerNameTimeStamp:
return self.logger
if self.logger:
while len(self.logger.handlers) > 0:
self.logger.handlers.pop()
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.INFO)
formatter = logging.Formatter('[%(asctime)s] %(levelname)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
if os_name == 'nt':
handler = logging.StreamHandler(sys_stdout)
else:
handler = TimedRotatingFileHandler(os.path.join('/opt/my_folder', 'my_log.log'),
when='midnight', backupCount=7)
handler.setLevel(logging.INFO)
handler.setFormatter(formatter)
self.logger.addHandler(handler)
self.loggerNameTimeStamp = datetime.now().strftime("%Y-%m-%d")
return self.logger
def info(self, message:str):
self.get_logger().info(message)
def debug(self, message:str):
print("DEBUG: " + message)
def warning(self, message:str):
self.get_logger().warning(message)
def error(self, message:str):
self.get_logger().error(message)
instance = None
def __new__(cls):
if not _WrapperLog.instance:
_WrapperLog.instance = _WrapperLog.__Log()
return _WrapperLog.instance
def __getattr__(self, name):
return getattr(self._WrapperLog, name)
def __setattr__(self, name):
return setattr(self._WrapperLog, name)
Log = _WrapperLog()
example of what remains of the files from yesterday and the day before yesterday:
my_log.log.2022-04-11:
[2022-04-12 00:00:01] INFO: <censored>
[2022-04-12 00:00:01] INFO: <censored>
my_log.log.2022-04-10:
[2022-04-11 00:00:01] INFO: <censored>
[2022-04-11 00:00:01] INFO: <censored>
[2022-04-11 00:00:01] INFO: <this line started like this and stopped after INFO>
Related
I am new to Python and learning logging technique with Decorator.
For me the below code is not generating required log file. Debugged the code, getting correct message to logger statement but the file is not generating. From Test method i am call the required function where i have implemented Decorator. Please guide where i am doing mistake.
try:
import csv
import requests
import datetime
import os
import sys
import logging
except Exception as e:
print("Some Modules are missing {}".format(e))
class Meta(type):
""" Meta class"""
def __call__(cls, *args, **kwargs):
instance = super(Meta, cls).__call__(*args, **kwargs)
return instance
def __init__(cls, name, base, attr):
super(Meta, cls).__init__(name, base, attr)
class log(object):
def __init__(self, func):
self.func = func
def __call__(self, *args, **kwargs):
""" Wrapper Function"""
start = datetime.datetime.now() #start time
Tem = self.func(*args) #call Function
Argument = args
FunName = self.func.__name__ #get Function name
end = datetime.datetime.now() #end Time
message = """
Function : {}
Execustion Time : {}
Argument : {}
Memory : {} Bytes
Date : {}
""".format(FunName,
end-start,
Argument,
sys.getsizeof(self.func),
start
)
cwd = os.getcwd();
folder = 'Logs'
newPath = os.path.join(cwd, folder)
try:
"""Try to create a folder """
os.mkdir(newPath)
except:
"""Folder already exist """
logging.basicConfig(filename='apiRun.log'.format(newPath), level=logging.DEBUG)
logging.debug(message)
return Tem
class APIHelper(metaclass=Meta):
def __init__(self, *args, **kwargs):
pass
#log
def star_wars_characters(url):
#self.url = url
api_response = requests.get(url)
people = []
if api_response.status_code == 200:
data = api_response.json()
for d in data['results']:
character = []
character.append(d['name'])
character.append(d['height'])
character.append(d['gender'])
people.append(character)
return people
else:
return "Bad Request"
My Test Method:
import unittest
import csv
from com.Script.APIHelper import APIHelper
class TestAPI(unittest.TestCase):
def _setUp(self, file_name):
self.api = APIHelper()
with open(file_name, "w") as self.fd:
self.csvfile = csv.writer(self.fd, delimiter = ',')
self.csvfile.writerow(['Name','Height','Gender'])
def tearDown(self):
self.fd.close()
def test_responseNotEmpty(self):
file_name = 'SWAPI.csv'
self._setUp(file_name)
people = self.api.star_wars_characters("https://swapi.dev/api/people/")
assert type(people) is list
Thanks you in Advance.
Add finally
Change filename='apiRun.log' to filename='{}/apiRun.log'
try:
"""Try to create a folder """
os.mkdir(newPath)
except:
"""Folder already exist """
finally:
logging.basicConfig(filename='{}/apiRun.log'.format(newPath), level=logging.DEBUG)
logging.debug(message)
except is executed only when an exception is raised from try.
finally is always executed.
I have a module called myLog.py which is being accessed by multiple other modules in a project. The myLog.py module has two handlers: file_handler that inputs logs into file and stream_handler that outputs logs to a console. For modules where no threading is occurring i.e myLog.py is only being accessed by a single process the logs are being inserted properly but for modules where threading is being implemented i.e myLog.py is being accessed by multiple processes at the same time I am getting multiple logs of the same line being inserted in my log_file.txt.
While going through logging documentation I found out that logging module is thread_safe but my implementation says things differently. How should I initialize the function setLogger() in myLog.py such that if it gets accessed by multiple threads at the same time it gives the correct output?
#myLog.py
#setup of logger
def setLogger(logfile_name = "log_file.txt"):
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
formatter = logging.Formatter('%(message)s')
file_handler = logging.FileHandler(logfile_name)
file_handler.setFormatter(formatter)
stream_handler = logging.StreamHandler()
logger.addHandler(file_handler)
logger.addHandler(stream_handler)
return logger
So suppose for example it is being accessed by a module called parser.py which implements threading then the log statements get printed out in a very random duplicated fashion.
#parser.py
import threading
import myLog
logger = myLog.setLogger()
class investigate(threading.Thread):
def __init__(self, section, file, buffer, args):
threading.Thread.__init__(self)
self.section = section
self.file = file
self.buffer = buffer
self.args = args
self.sig = self.pub = None
self.exc = None
def run(self):
aprint("Starting section %d file %d" % (self.section, self.file))
self.exc = None
try:
self.sign()
aprint("Done section %d file %d" % (self.section, self.file))
except:
self.exc = sys.exc_info()
def sign(self):
self.sig, self.pub = sign_hsm(self.buffer, self.args)
if self.sig is None or self.pub is None:
raise Exception("Empty signing result")
def store(self, bot):
sec = filter(lambda x: x.type == self.section, bot.sections)[0]
if self.file == 0xFF:
signature = sec.signature
else:
signature = sec.files[self.file].signature
signature.sig = self.sig
signature.pub = self.pub
def join(self, *args, **kwargs):
threading.Thread.join(self, *args, **kwargs)
if self.exc:
msg = "Thread '%s' threw an exception: %s" % (self.getName(), self.exc[1])
new_exc = Exception(msg)
raise new_exc.__class__, new_exc, self.exc[2]
def PrintVersion():
logger.info("This is output.")
print_lock = threading.RLock()
def aprint(*args, **kwargs):
if verbosityLevel > 0:
with print_lock:
return logger.info(*args, **kwargs)
def multipleTimes():
logger.info("Multiple times.")
if __name__ == "__main__":
PrintVersion()
for investigate in investigations:
investigate.start()
.......
.......
.......
logger.info("This gets repeated")
multipleTimes()
So since multiple threads are trying to access the setLogger() I get logger.info() outputs such as:
This is output.
This is output.
This is output.
This is output.
This is output.
This gets repeated.
This gets repeated.
This gets repeated.
Multiple times.
Multiple times.
Multiple times.
Multiple times.
Multiple times.
Multiple times.
What I should be getting:
This is output.
This gets repeated.
Multiple times.
I have a custom logger class that worked fine in one module, but if I use it in other module of the same package and with the same filehandler it multiplies record from 1 to n times.
I use it in 5 threads that's why I'm trying to close filehandler each time message was recorded
class RootLogger():
def __init__(self):
self.logger = None
# skeleton for creating custom logger
# name - name of the logger, string or class/module name
def set_config(self, name, logfile):
self.logger = logging.getLogger(name)
# format of the message
formatter = logging.Formatter('%(asctime)s - [%(name)s] [%(levelname)s] %(message)s')
# log file rotation, time
handler = RotatingFileHandler(filename=logfile,
maxBytes=8000000,
backupCount=5,
encoding='utf-8')
#handler = StreamHandler()
handler.setFormatter(formatter)
self.logger.setLevel(logging.DEBUG)
self.logger.addHandler(handler)
self.logger.setLevel(logging.DEBUG)
return self.logger
# patch for excluding 'too many opened files' error
def __del__(self):
if self.logger:
for hdlr in self.logger.handlers:
#hdlr.flush()
hdlr.close()
# override commands
def info(self, msg):
self.logger.info(msg)
self.__del__()
def debug(self, msg):
self.logger.debug(msg)
self.__del__()
def error(self, msg):
self.logger.error(msg)
self.__del__()
def warning(self, msg):
self.logger.warning(msg)
self.__del__()
def critical(self, msg):
self.logger.warning(msg)
self.__del__()
You want to check if handlers already exist for the logger before creating more handlers.
self.logger = logging.getLogger(name)
if not self.logger.handlers:
#Create the handlers
I have a python pipeline code which calls around 10 to 12 python classes.
I wanted to know and log time consumed by each method in each class.
Could anyone suggest some methods to do this in python
I'm using python inbuilt logging module for logging
import logging
import time
class myClass():
logErrors = False
logger = None
def log(self, msg):
if self.logger != None:
self.logger.warning(msg)
def __init__(self, log=False):
self.logErrors = log
if self.logErrors == True:
self.logger = logging.getLogger("myClass")
self.logger.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
self.ch = logging.StreamHandler()
self.ch.setLevel(logging.INFO)
self.ch.setFormatter(formatter)
self.logger.addHandler(self.ch)
def method1(self):
# log start time of method
self.log("method1() started")
# code
print("foobar")
time.sleep(1)
# log end time of method
self.log("method1() ended")
my = myClass(log=True)
my.method1()
I have a class called Job which has a logger
class MyFileHandler(logging.FileHandler):
def __init__(self, filename):
self.filename = filename
super(MyFileHandler, self).__init__(filename)
def emit(self, record):
log_text = self.format(record)
try:
fh = open(self.filename, "a")
fh.write("%s\n" % log_text)
fh.close()
return True
except:
return False
log_formatter = logging.Formatter('br: %(message)s')
class Job(object):
def __init__(self, name):
self.name = name
self.logger = logging.getLogger(self.name)
log_hdlr = MyFileHandler('/tmp/%s' % name)
log_hdlr.setFormatter(log_formatter)
self.logger.addHandler(log_hdlr)
self.logger.setLevel(logging.INFO)
jobs = []
for i in range(100):
j = Job(str(i))
job.append(j)
and jobs go off do something and logs via job.logger.info()
but when i have multiple jobs i.e. thousands, it's throwing error
IOError: [Errno 24]
Too many open files:
'/x/prototype_3885946_1608131132/cm/cm_conv/logs/20160827-195925.log'
I thought every time I logged something, it would open then close the file as I have overwritten emit()
Is there a pattern/ways to have thousands of loggers?
Guess your operating system is running out of file handles.
Fyi, instead of executing self.logger.info(msg) directly, i just wrapped it around the following code which opens filehandler and closes it each time im writing to a log..
rewrite self.logger.info(msg) to self.write_to_log(msg)
where:
def write_to_log(self, msg):
log_hdlr = MyFileHandler('/tmp/%s' % self.name)
log_hdlr.setFormatter(log_formatter)
self.logger.addHandler(log_hdlr)
self.logger.setLevel(logging.INFO)
self.logger.info(msg) # <----- actually calling .info() here
for handler in self.logger.handlers:
handler.close()
self.logger.removeHandler(handler)