I have a module called myLog.py which is being accessed by multiple other modules in a project. The myLog.py module has two handlers: file_handler that inputs logs into file and stream_handler that outputs logs to a console. For modules where no threading is occurring i.e myLog.py is only being accessed by a single process the logs are being inserted properly but for modules where threading is being implemented i.e myLog.py is being accessed by multiple processes at the same time I am getting multiple logs of the same line being inserted in my log_file.txt.
While going through logging documentation I found out that logging module is thread_safe but my implementation says things differently. How should I initialize the function setLogger() in myLog.py such that if it gets accessed by multiple threads at the same time it gives the correct output?
#myLog.py
#setup of logger
def setLogger(logfile_name = "log_file.txt"):
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
formatter = logging.Formatter('%(message)s')
file_handler = logging.FileHandler(logfile_name)
file_handler.setFormatter(formatter)
stream_handler = logging.StreamHandler()
logger.addHandler(file_handler)
logger.addHandler(stream_handler)
return logger
So suppose for example it is being accessed by a module called parser.py which implements threading then the log statements get printed out in a very random duplicated fashion.
#parser.py
import threading
import myLog
logger = myLog.setLogger()
class investigate(threading.Thread):
def __init__(self, section, file, buffer, args):
threading.Thread.__init__(self)
self.section = section
self.file = file
self.buffer = buffer
self.args = args
self.sig = self.pub = None
self.exc = None
def run(self):
aprint("Starting section %d file %d" % (self.section, self.file))
self.exc = None
try:
self.sign()
aprint("Done section %d file %d" % (self.section, self.file))
except:
self.exc = sys.exc_info()
def sign(self):
self.sig, self.pub = sign_hsm(self.buffer, self.args)
if self.sig is None or self.pub is None:
raise Exception("Empty signing result")
def store(self, bot):
sec = filter(lambda x: x.type == self.section, bot.sections)[0]
if self.file == 0xFF:
signature = sec.signature
else:
signature = sec.files[self.file].signature
signature.sig = self.sig
signature.pub = self.pub
def join(self, *args, **kwargs):
threading.Thread.join(self, *args, **kwargs)
if self.exc:
msg = "Thread '%s' threw an exception: %s" % (self.getName(), self.exc[1])
new_exc = Exception(msg)
raise new_exc.__class__, new_exc, self.exc[2]
def PrintVersion():
logger.info("This is output.")
print_lock = threading.RLock()
def aprint(*args, **kwargs):
if verbosityLevel > 0:
with print_lock:
return logger.info(*args, **kwargs)
def multipleTimes():
logger.info("Multiple times.")
if __name__ == "__main__":
PrintVersion()
for investigate in investigations:
investigate.start()
.......
.......
.......
logger.info("This gets repeated")
multipleTimes()
So since multiple threads are trying to access the setLogger() I get logger.info() outputs such as:
This is output.
This is output.
This is output.
This is output.
This is output.
This gets repeated.
This gets repeated.
This gets repeated.
Multiple times.
Multiple times.
Multiple times.
Multiple times.
Multiple times.
Multiple times.
What I should be getting:
This is output.
This gets repeated.
Multiple times.
Related
I have a problem with the python logging library. I have a flask application that uses the following class (_WrapperLog) to write logs to a file, the library correctly writes logs to today's file and cleans up files older than 7 days correctly. the problem is that when the log rotation occurs, the old files remain saved but their content disappears (as in the example below), only the first 2/4 lines of the file remain, and then nothing, what could be due ?
import logging
from logging.handlers import TimedRotatingFileHandler
import os
from datetime import datetime
from os import name as os_name
from sys import stdout as sys_stdout
class _WrapperLog(object):
class __Log:
logger = None
loggerNameTimeStamp=None
def __init__(self):
self.logger = self.get_logger()
def __str__(self):
return str(self)
def get_logger(self):
if datetime.now().strftime("%Y-%m-%d") == self.loggerNameTimeStamp:
return self.logger
if self.logger:
while len(self.logger.handlers) > 0:
self.logger.handlers.pop()
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.INFO)
formatter = logging.Formatter('[%(asctime)s] %(levelname)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
if os_name == 'nt':
handler = logging.StreamHandler(sys_stdout)
else:
handler = TimedRotatingFileHandler(os.path.join('/opt/my_folder', 'my_log.log'),
when='midnight', backupCount=7)
handler.setLevel(logging.INFO)
handler.setFormatter(formatter)
self.logger.addHandler(handler)
self.loggerNameTimeStamp = datetime.now().strftime("%Y-%m-%d")
return self.logger
def info(self, message:str):
self.get_logger().info(message)
def debug(self, message:str):
print("DEBUG: " + message)
def warning(self, message:str):
self.get_logger().warning(message)
def error(self, message:str):
self.get_logger().error(message)
instance = None
def __new__(cls):
if not _WrapperLog.instance:
_WrapperLog.instance = _WrapperLog.__Log()
return _WrapperLog.instance
def __getattr__(self, name):
return getattr(self._WrapperLog, name)
def __setattr__(self, name):
return setattr(self._WrapperLog, name)
Log = _WrapperLog()
example of what remains of the files from yesterday and the day before yesterday:
my_log.log.2022-04-11:
[2022-04-12 00:00:01] INFO: <censored>
[2022-04-12 00:00:01] INFO: <censored>
my_log.log.2022-04-10:
[2022-04-11 00:00:01] INFO: <censored>
[2022-04-11 00:00:01] INFO: <censored>
[2022-04-11 00:00:01] INFO: <this line started like this and stopped after INFO>
I have a class called Job which has a logger
class MyFileHandler(logging.FileHandler):
def __init__(self, filename):
self.filename = filename
super(MyFileHandler, self).__init__(filename)
def emit(self, record):
log_text = self.format(record)
try:
fh = open(self.filename, "a")
fh.write("%s\n" % log_text)
fh.close()
return True
except:
return False
log_formatter = logging.Formatter('br: %(message)s')
class Job(object):
def __init__(self, name):
self.name = name
self.logger = logging.getLogger(self.name)
log_hdlr = MyFileHandler('/tmp/%s' % name)
log_hdlr.setFormatter(log_formatter)
self.logger.addHandler(log_hdlr)
self.logger.setLevel(logging.INFO)
jobs = []
for i in range(100):
j = Job(str(i))
job.append(j)
and jobs go off do something and logs via job.logger.info()
but when i have multiple jobs i.e. thousands, it's throwing error
IOError: [Errno 24]
Too many open files:
'/x/prototype_3885946_1608131132/cm/cm_conv/logs/20160827-195925.log'
I thought every time I logged something, it would open then close the file as I have overwritten emit()
Is there a pattern/ways to have thousands of loggers?
Guess your operating system is running out of file handles.
Fyi, instead of executing self.logger.info(msg) directly, i just wrapped it around the following code which opens filehandler and closes it each time im writing to a log..
rewrite self.logger.info(msg) to self.write_to_log(msg)
where:
def write_to_log(self, msg):
log_hdlr = MyFileHandler('/tmp/%s' % self.name)
log_hdlr.setFormatter(log_formatter)
self.logger.addHandler(log_hdlr)
self.logger.setLevel(logging.INFO)
self.logger.info(msg) # <----- actually calling .info() here
for handler in self.logger.handlers:
handler.close()
self.logger.removeHandler(handler)
I have an app with some subprocess running and I have successfully set the sys.excepthook exception handling for the main process. Now, I want to set it for the same hook on the subprocesses. I would expect it to be as simple copying the exact lines of code I used on the main process and that's it but it didn't work.
Next is my code:
class Consumer(multiprocessing.Process):
def __init__(self, codec_status_queue, logger_queue):
multiprocessing.Process.__init__(self)
self.codec_status_queue = codec_status_queue
self.logger_queue = logger_queue
def run(self):
# Set default unhandled exceptions handler
uncaughtErrorHandler = UncaughtErrorHandler(self.logger_queue)
sys.excepthook = uncaughtErrorHandler.error_handler
1/0
class UncaughtErrorHandler(object):
def __init__(self, logger_queue, child_processes=None):
self.logger_queue = logger_queue
self.child_processes = child_processes
def error_handler(self, type, value, trace_back):
trace_formatted = "".join(traceback.format_tb(trace_back))
exeption_message = "Unhandled Exception:\n Type: %s\n Value: %s\n Line: %s\n Traceback:\n %s" % (type, value.message, trace_back.tb_lineno, trace_formatted)
logger_queue.put((LoggerThread.CRITICAL, exeption_message))
if self.child_processes:
self.stop_children()
# Stopping this process
sys.exit()
def stop_children(self):
num_children = len(self.child_processes)
logger_queue.put((LoggerThread.DEBUG, "Terminating child processes (%s)" % num_children))
for process in self.child_processes:
log_message = "Terminating %s with PID %s" % (process.name, process.pid)
logger_queue.put((LoggerThread.DEBUG, log_message))
process.terminate()
if __name__ == '__main__':
...
# Create processes and communication queues
codec_status_queue = multiprocessing.Queue()
num_consumers = multiprocessing.cpu_count() * 2
print 'Creating %d consumers' % num_consumers
consumers = [ Consumer(codec_status_queue, logger_queue)
for i in xrange(num_consumers) ]
# Set default unhandled exceptions handler
uncaughtErrorHandler = UncaughtErrorHandler(logger_queue, consumers)
sys.excepthook = uncaughtErrorHandler.error_handler
# Start processes
for consumer in consumers:
consumer.daemon = True
consumer.start()
If I put the 1/0 on the __main__ part the UncaughtErrorHandler catches the exception but when the 1/0 is put as shown above, it doesn't.
Maybe someone can tell me what am I doing wrong?
The following code was written for Python 3.x but can be adapted to work with Python 3.x instead. It provides an alternative solution to overriding sys.excepthook in child processes. A simple fix involves catching all exceptions and handing the data from sys.exc_info over to the exception handler. The main process could use a similar pattern for exceptions but retains the original design from your program. The example shown below should be a full working demonstration you can play around with and adapt to your needs.
#! /usr/bin/env python3
import logging
import multiprocessing
import queue
import sys
import threading
import time
import traceback
def main():
"""Demonstrate exception handling and logging in several processes."""
logger_queue = multiprocessing.Queue()
logger_thread = LoggerThread(logger_queue)
logger_thread.start()
try:
# Create processes and communication queues
codec_status_queue = multiprocessing.Queue()
num_consumers = multiprocessing.cpu_count() * 2
print('Creating {} consumers'.format(num_consumers))
consumers = [Consumer(codec_status_queue, logger_queue)
for _ in range(num_consumers)]
# Set default unhandled exceptions handler
uncaught_error_handler = UncaughtErrorHandler(logger_queue, consumers)
sys.excepthook = uncaught_error_handler.error_handler
# Start processes
for consumer in consumers:
consumer.start()
time.sleep(2)
finally:
logger_thread.shutdown()
def get_message(value):
"""Retrieve an exception's error message and return it."""
if hasattr(value, 'message'):
return value.message
if hasattr(value, 'args') and value.args:
return value.args[0]
class LoggerThread(threading.Thread):
"""Handle logging messages coming from various sources via a queue."""
CRITICAL = logging.CRITICAL
DEBUG = logging.DEBUG
def __init__(self, logger_queue):
"""Initialize an instance of the LoggerThread class."""
super().__init__()
self.logger_queue = logger_queue
self.mutex = threading.Lock()
self.running = False
def run(self):
"""Process messages coming through the queue until shutdown."""
self.running = True
while self.running:
try:
while True:
self.handle_message(*self.logger_queue.get(True, 0.1))
except queue.Empty:
pass
def handle_message(self, level, message):
"""Show the message while ensuring a guaranteed order on screen."""
with self.mutex:
print('Level:', level)
print('Message:', message)
print('=' * 80, flush=True)
def shutdown(self):
"""Signal the thread to exit once it runs out of messages."""
self.running = False
class Consumer(multiprocessing.Process):
"""Simulate a consumer process that handles data from a queue."""
def __init__(self, codec_status_queue, logger_queue):
"""Initialize an instance of the Consumer class."""
super().__init__()
self.codec_status_queue = codec_status_queue
self.logger_queue = logger_queue
self.daemon = True
def run(self):
"""Begin working as a consumer while handling any exceptions."""
# Set default unhandled exceptions handler
uncaught_error_handler = UncaughtErrorHandler(self.logger_queue)
try:
self.do_consumer_work()
except:
uncaught_error_handler.error_handler(*sys.exc_info())
def do_consumer_work(self):
"""Pretend to be doing the work of a consumer."""
junk = 1 / 0
print('Process', self.ident, 'calculated', junk)
class UncaughtErrorHandler:
"""Organize error handling to automatically terminate child processes."""
def __init__(self, logger_queue, child_processes=None):
"""Initialize an instance of the UncaughtErrorHandler class."""
self.logger_queue = logger_queue
self.child_processes = child_processes
def error_handler(self, kind, value, trace_back):
"""Record errors as they happen and terminate the process tree."""
trace_formatted = ''.join(traceback.format_tb(trace_back))
exception_message = ('Unhandled Exception:\n'
' Type: {}\n'
' Value: {}\n'
' Line: {}\n'
' Traceback:\n{}').format(
kind, get_message(value), trace_back.tb_lineno, trace_formatted)
self.logger_queue.put((LoggerThread.CRITICAL, exception_message))
if self.child_processes:
self.stop_children()
# Stopping this process
sys.exit()
def stop_children(self):
"""Terminate all children associated with this error handler."""
num_children = len(self.child_processes)
log_message = 'Terminating child processes({})'.format(num_children)
self.logger_queue.put((LoggerThread.DEBUG, log_message))
for process in self.child_processes:
log_message = 'Terminating {} with PID {}'.format(
process.name, process.pid)
self.logger_queue.put((LoggerThread.DEBUG, log_message))
process.terminate()
if __name__ == '__main__':
main()
My questions are interlaced within my code:
#!/usr/bin/python
import threading
import logging, logging.handlers
import hpclib
import json
import time
from datetime import datetime
from features import *
import sys
if len(sys.argv) != 3:
print "Please provide the correct inputs"
print "Usage: rest_test.py <controllerip> <counter>"
sys.exit()
controller = sys.argv[1]
counter = int(sys.argv[2])
class FuncThread(threading.Thread):
def __init__(self, target, *args):
self._target = target
self._args = args
threading.Thread.__init__(self)
def run(self):
self._target(*self._args)
def datapath_thread(ipaddress, testlogfile,count):
#initialize logging system
testlogger = logging.getLogger("testlogger")
testlogger.setLevel(logging.DEBUG)
file = open(testlogfile,'w')
file.close()
# This handler writes everything to a file.
h1 = logging.FileHandler(testlogfile)
f = logging.Formatter("%(levelname)s %(asctime)s %(funcName)s %(lineno)d %(message)s")
h1.setFormatter(f)
h1.setLevel(logging.DEBUG)
testlogger.addHandler(h1)
mylib = hpclib.hpclib(ipaddress)
success_count = 0
failure_count = 0
for i in range(count):
t1=datetime.now()
try:
(code, val) = datapaths.listDatapaths(mylib)
I want to pass this function datapaths.listDatapaths(mylib) as a argument from a thread below, something like (code,val)=functionname
if code == 200:
success_count +=1
else:
testlogger.debug("Return Code other than 200 received with code = %d, value = %s"%(code,val))
failure_count +=1
except:
failure_count += 1
testlogger.debug ("Unexpected error: %s"%sys.exc_info()[0])
continue
t2=datetime.now()
diff=t2-t1
testlogger.debug('RETURN code: %d. Time taken in sec = %s,Iteration = %d, Success = %d, Failure = %d'%(code,diff.seconds,i+1,success_count,failure_count))
time.sleep(1)
testlogger.removeHandler(h1)
# Passing ipadress of controller and log file name
t1 = FuncThread(datapath_thread, controller, "datapaths.log",counter)
Here I would like to pass function name as one of the argument,something like t1 = FuncThread(datapath_thread, controller, datapaths.listDatapaths(mylib),"datapaths.log",counter)
t1.start()
t1.join()
I have many functions to call like this,so want a easy way to call all the functions from one single function using many threads
Firstly, FuncThread is not very useful - FuncThread(func, *args) can be spelt Thread(target=lambda: func(*args)) or Thread(target=func, args=args).
You're pretty close - instead of passing in the result of calling the function, pass in the function itself
def datapath_thread(ipaddress, test_func, testlogfile, count):
# ...
for i in range(count):
# ...
try:
(code, val) = test_func(mylib)
#...
thread = Thread(target=datapath_thread, args=(
controller,
datapaths.listDatapaths,
"datapaths.log",
counter
))
I am trying to find a good way to log a warning message but appending to it information that is only known by the caller of the function.
I think it will be clear with an example.
# log method as parameter
class Runner1(object):
def __init__(self, log):
self.log = log
def run(self):
self.log('First Warning')
self.log('Second Warning')
return 42
class Main1(object):
def __init__(self):
self._runner = Runner1(self.log)
def log(self, message):
print('Some object specific info: {}'.format(message))
def run(self):
print(self._runner.run())
e1 = Main1()
e1.run()
The Main object has a log function that adds to any message its own information before logging it. This log function is given as a parameter (in this case to a Runner object). Carrying this extra parameter all the time is extremely annoying and I would like to avoid it. There are usually lots of object/functions and therefore I have discarded the use of the logging method as I would need to create a different logger for each object. (Is this correct?)
I have tried to bubble the warning using the warning module:
# warning module
import warnings
class Runner2(object):
def run(self):
warnings.warn('First Warning')
warnings.warn('Second Warning')
return 42
class Main2(object):
def __init__(self):
self._runner = Runner2()
def log(self, message):
print('Some object specific info: {}'.format(message))
def run(self):
with warnings.catch_warnings(record=True) as ws:
warnings.simplefilter("always")
out = self._runner.run()
for w in ws:
self.log(w.message)
print(out)
e2 = Main2()
e2.run()
But according to the docs, this is not thread safe.
Finally, I have also tried some generators:
# yield warning
class _Warning(object):
def __init__(self, message):
self.message = message
class Runner3(object):
def run(self):
yield _Warning('First Warning')
yield _Warning('Second Warning')
yield 42
class Main3(object):
def __init__(self):
self._runner = Runner3()
def log(self, message):
print('Some object specific info: {}'.format(message))
def run(self):
for out in self._runner.run():
if not isinstance(out, _Warning):
break
self.log(out.message)
print(out)
e3 = Main3()
e3.run()
But the fact that you have to modify the Runner.run to yield (instead of return) the final result is inconvenient as functions will have to be specifically changed to be used in this way (Maybe this will change in the future? Last QA in PEP255). Additionally, I am not sure if there is any other trouble with this type of implementation.
So what I am looking for is a thread-safe way of bubbling warnings that does not require passing parameters. I also would like that methods that do not have warnings remain unchanged. Adding a special construct such as yield or warning.warn to bubble the warnings would be fine.
Any ideas?
import Queue
log = Queue.Queue()
class Runner1(object):
def run(self):
log.put('First Warning')
log.put('Second Warning')
return 42
class Main1(object):
def __init__(self):
self._runner = Runner1()
def log(self, message):
print('Some object specific info: {0}'.format(message))
def run(self):
out=self._runner.run()
while True:
try:
msg = log.get_nowait()
self.log(msg)
except Queue.Empty:
break
print(out)
e1 = Main1()
e1.run()
yields
Some object specific info: First Warning
Some object specific info: Second Warning
42